client2
2016 January 13, 17:05
fio-2.1.10
CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
60 s ramp + 7200 s run, per test; s between tests

# 
# SR-71mach5 Configuration
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 

Genuine Intel(R) CPU @ 2.90GHz     Status: Populated, Enabled Core Enabled: 8
Genuine Intel(R) CPU @ 2.90GHz     Status: Populated, Enabled Core Enabled: 8
DIMM_A1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38B42B22  
DIMM_B1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38542B15  
DIMM_C1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38742B21  
DIMM_D1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38142B2C  
DIMM_E1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38142B0E  
DIMM_F1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38242B11  
DIMM_G1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38642B24  
DIMM_H1: 4096 MB | 64/72 bits | 1600 MHz | Hynix            | HMT351R7CFR8C-PB | SN: 38342B20  
____________________________________________________________________
MemTotal:       32673256 kB


# 
# actual mount paramters
# 

10.0.5.144:/control on /root/sr71/control type nfs4 (rw,relatime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=10.0.5.133,local_lock=none,addr=10.0.5.144)
10.10.2.100:/sr71/C0P2 on /root/sr71/SATA0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.2.10,local_lock=none,addr=10.10.2.100)
10.10.3.100:/sr71/C1P2 on /root/sr71/SATA1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.3.10,local_lock=none,addr=10.10.3.100)
10.10.2.100:/sr71/NVMeR5V0P2 on /root/sr71/NVMe0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.2.10,local_lock=none,addr=10.10.2.100)
10.10.3.100:/sr71/NVMeR5V1P2 on /root/sr71/NVMe1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.3.10,local_lock=none,addr=10.10.3.100)



# 
# fio job file
fio-2.1.10
# 

 seconds between fio tests
[global]
ioengine=libaio
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
group_reporting

[SATA-R5-01]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA0/fio.tst

[SATA-R5-11]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA1/fio.tst

[NVMe-R5-01]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe0/fio.tst

[NVMe-R5-11]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe1/fio.tst




# 
# 201601131705 test details
# 



# 
# Wed 13 Jan 2016 05:05:00 PM EST
# 8kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 8kB randwrite with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 2922 IOPS | 23 MBps | 340 μsec Average Latency 
SATA-R5-1: 2751 IOPS | 22 MBps | 361 μsec Average Latency 
NVMe-R5-0: 5064 IOPS | 40 MBps | 393 μsec Average Latency 
NVMe-R5-1: 5435 IOPS | 43 MBps | 365 μsec Average Latency 

Totals
IOPS: 16172
MBPS: 128
Average Latency (μsec): 364

SATA-R5-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=162130: Wed Jan 13 19:06:01 2016
  write: io=164410MB, bw=23383KB/s, iops=2922, runt=7200001msec
    slat (usec): min=3, max=455, avg=11.59, stdev= 5.64
    clat (usec): min=0, max=200048, avg=328.59, stdev=338.60
     lat (usec): min=99, max=200063, avg=340.33, stdev=338.00
    clat percentiles (usec):
     |  1.00th=[  107],  5.00th=[  122], 10.00th=[  219], 20.00th=[  251],
     | 30.00th=[  350], 40.00th=[  362], 50.00th=[  362], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  370], 95.00th=[  382],
     | 99.00th=[  414], 99.50th=[  458], 99.90th=[  516], 99.95th=[  636],
     | 99.99th=[ 1368]
    bw (KB  /s): min=    0, max=38768, per=100.00%, avg=23390.93, stdev=2369.35
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.09%, 250=19.72%, 500=80.02%
    lat (usec) : 750=0.12%, 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.89%, sys=2.44%, ctx=42423475, majf=0, minf=1437
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=21044543/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=162131: Wed Jan 13 19:06:01 2016
  write: io=154755MB, bw=22010KB/s, iops=2751, runt=7200001msec
    slat (usec): min=4, max=2525, avg=23.22, stdev=11.49
    clat (usec): min=3, max=200761, avg=337.56, stdev=85.47
     lat (usec): min=109, max=200785, avg=361.15, stdev=83.40
    clat percentiles (usec):
     |  1.00th=[  181],  5.00th=[  221], 10.00th=[  274], 20.00th=[  330],
     | 30.00th=[  338], 40.00th=[  346], 50.00th=[  350], 60.00th=[  350],
     | 70.00th=[  362], 80.00th=[  362], 90.00th=[  366], 95.00th=[  378],
     | 99.00th=[  438], 99.50th=[  470], 99.90th=[  524], 99.95th=[  628],
     | 99.99th=[ 1144]
    bw (KB  /s): min=    0, max=31664, per=100.00%, avg=22016.20, stdev=607.79
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=8.82%
    lat (usec) : 500=91.02%, 750=0.12%, 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.25%, sys=3.61%, ctx=39941953, majf=0, minf=1416
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19808633/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=162132: Wed Jan 13 19:06:01 2016
  write: io=284868MB, bw=40515KB/s, iops=5064, runt=7200001msec
    slat (usec): min=2, max=462, avg=10.52, stdev= 4.18
    clat (usec): min=1, max=200506, avg=382.88, stdev=1837.74
     lat (usec): min=96, max=200522, avg=393.54, stdev=1837.71
    clat percentiles (usec):
     |  1.00th=[  189],  5.00th=[  239], 10.00th=[  294], 20.00th=[  354],
     | 30.00th=[  362], 40.00th=[  362], 50.00th=[  366], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  374], 95.00th=[  390],
     | 99.00th=[  430], 99.50th=[  478], 99.90th=[  668], 99.95th=[17280],
     | 99.99th=[91648]
    bw (KB  /s): min=    0, max=51472, per=100.00%, avg=40613.44, stdev=5452.14
    lat (usec) : 2=0.01%, 100=0.01%, 250=6.77%, 500=92.97%, 750=0.16%
    lat (usec) : 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.03%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.23%, sys=4.00%, ctx=72477724, majf=0, minf=1710
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=36463114/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=162133: Wed Jan 13 19:06:01 2016
  write: io=305770MB, bw=43487KB/s, iops=5435, runt=7200001msec
    slat (usec): min=3, max=810, avg=21.56, stdev=10.91
    clat (usec): min=1, max=201048, avg=343.84, stdev=474.15
     lat (usec): min=105, max=201071, avg=365.74, stdev=473.87
    clat percentiles (usec):
     |  1.00th=[  191],  5.00th=[  235], 10.00th=[  294], 20.00th=[  330],
     | 30.00th=[  342], 40.00th=[  350], 50.00th=[  350], 60.00th=[  354],
     | 70.00th=[  362], 80.00th=[  362], 90.00th=[  366], 95.00th=[  378],
     | 99.00th=[  438], 99.50th=[  470], 99.90th=[  540], 99.95th=[  708],
     | 99.99th=[ 1576]
    bw (KB  /s): min=    0, max=46864, per=100.00%, avg=43502.65, stdev=1821.98
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=6.74%
    lat (usec) : 500=93.07%, 750=0.14%, 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=2.19%, sys=6.70%, ctx=77693997, majf=0, minf=1809
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=39138577/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=164410MB, aggrb=23382KB/s, minb=23382KB/s, maxb=23382KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=154755MB, aggrb=22009KB/s, minb=22009KB/s, maxb=22009KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=284868MB, aggrb=40514KB/s, minb=40514KB/s, maxb=40514KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=305770MB, aggrb=43487KB/s, minb=43487KB/s, maxb=43487KB/s, mint=7200001msec, maxt=7200001msec



# 
# Wed 13 Jan 2016 07:06:03 PM EST
# 64kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 64kB randwrite with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1233 IOPS | 78 MBps | 805 μsec Average Latency 
SATA-R5-1: 1390 IOPS | 89 MBps | 716 μsec Average Latency 
NVMe-R5-0: 597 IOPS | 38 MBps | 13395 μsec Average Latency 
NVMe-R5-1: 652 IOPS | 41 MBps | 12260 μsec Average Latency 

Totals
IOPS: 3872
MBPS: 246
Average Latency (μsec): 6794

SATA-R5-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 10 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=167293: Wed Jan 13 21:07:04 2016
  write: io=555071MB, bw=78943KB/s, iops=1233, runt=7200001msec
    slat (usec): min=10, max=1406, avg=58.38, stdev=13.26
    clat (usec): min=2, max=3978.2K, avg=746.55, stdev=9253.71
     lat (usec): min=228, max=3978.2K, avg=805.26, stdev=9253.77
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  310], 10.00th=[  314], 20.00th=[  330],
     | 30.00th=[  346], 40.00th=[  414], 50.00th=[  434], 60.00th=[  450],
     | 70.00th=[  478], 80.00th=[  548], 90.00th=[  564], 95.00th=[  588],
     | 99.00th=[  700], 99.50th=[  996], 99.90th=[100864], 99.95th=[158720],
     | 99.99th=[395264]
    bw (KB  /s): min=    1, max=152832, per=100.00%, avg=81859.13, stdev=25463.44
    lat (usec) : 4=0.01%, 250=0.02%, 500=71.80%, 750=27.36%, 1000=0.32%
    lat (msec) : 2=0.14%, 4=0.03%, 10=0.02%, 20=0.01%, 50=0.11%
    lat (msec) : 100=0.07%, 250=0.08%, 500=0.02%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.57%, sys=3.95%, ctx=17928088, majf=0, minf=3342
  IO depths    : 1=100.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=8881130/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=167294: Wed Jan 13 21:07:04 2016
  write: io=625897MB, bw=89016KB/s, iops=1390, runt=7200001msec
    slat (usec): min=11, max=287, avg=33.45, stdev= 9.86
    clat (usec): min=121, max=3949.4K, avg=682.81, stdev=8749.76
     lat (usec): min=195, max=3949.4K, avg=716.48, stdev=8749.79
    clat percentiles (usec):
     |  1.00th=[  294],  5.00th=[  314], 10.00th=[  326], 20.00th=[  338],
     | 30.00th=[  342], 40.00th=[  346], 50.00th=[  346], 60.00th=[  358],
     | 70.00th=[  378], 80.00th=[  442], 90.00th=[  470], 95.00th=[  532],
     | 99.00th=[  660], 99.50th=[  908], 99.90th=[96768], 99.95th=[150528],
     | 99.99th=[354304]
    bw (KB  /s): min=    1, max=168192, per=100.00%, avg=92232.60, stdev=30512.96
    lat (usec) : 250=0.02%, 500=93.68%, 750=5.61%, 1000=0.24%
    lat (msec) : 2=0.11%, 4=0.02%, 10=0.01%, 20=0.02%, 50=0.12%
    lat (msec) : 100=0.08%, 250=0.08%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.10%, sys=2.59%, ctx=20206412, majf=0, minf=3360
  IO depths    : 1=100.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10014347/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=167295: Wed Jan 13 21:07:04 2016
  write: io=268672MB, bw=38210KB/s, iops=597, runt=7200168msec
    slat (usec): min=7, max=338, avg=48.94, stdev=29.69
    clat (usec): min=109, max=3985.8K, avg=13346.77, stdev=53205.45
     lat (usec): min=242, max=3985.8K, avg=13395.97, stdev=53205.12
    clat percentiles (usec):
     |  1.00th=[  302],  5.00th=[  374], 10.00th=[  410], 20.00th=[  458],
     | 30.00th=[  494], 40.00th=[  532], 50.00th=[  564], 60.00th=[  604],
     | 70.00th=[  652], 80.00th=[  724], 90.00th=[ 1080], 95.00th=[130560],
     | 99.00th=[199680], 99.50th=[276480], 99.90th=[464896], 99.95th=[610304],
     | 99.99th=[1011712]
    bw (KB  /s): min=    1, max=132972, per=25.50%, avg=9741.92, stdev=6884.29
    lat (usec) : 250=0.22%, 500=31.96%, 750=49.79%, 1000=7.28%
    lat (msec) : 2=2.68%, 4=0.28%, 10=0.12%, 20=0.05%, 50=0.59%
    lat (msec) : 100=1.55%, 250=4.88%, 500=0.52%, 750=0.06%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.14%, sys=0.39%, ctx=8027244, majf=0, minf=20791
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4298743/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=167299: Wed Jan 13 21:07:04 2016
  write: io=293544MB, bw=41748KB/s, iops=652, runt=7200137msec
    slat (usec): min=7, max=370, avg=48.93, stdev=29.61
    clat (usec): min=74, max=3739.6K, avg=12211.50, stdev=50439.63
     lat (usec): min=243, max=3739.7K, avg=12260.67, stdev=50439.32
    clat percentiles (usec):
     |  1.00th=[  302],  5.00th=[  370], 10.00th=[  410], 20.00th=[  458],
     | 30.00th=[  490], 40.00th=[  524], 50.00th=[  556], 60.00th=[  596],
     | 70.00th=[  644], 80.00th=[  716], 90.00th=[ 1012], 95.00th=[94720],
     | 99.00th=[199680], 99.50th=[259072], 99.90th=[448512], 99.95th=[602112],
     | 99.99th=[987136]
    bw (KB  /s): min=    1, max=141937, per=25.59%, avg=10681.10, stdev=7288.21
    lat (usec) : 100=0.01%, 250=0.22%, 500=33.30%, 750=49.62%, 1000=6.70%
    lat (msec) : 2=2.51%, 4=0.27%, 10=0.11%, 20=0.05%, 50=0.68%
    lat (msec) : 100=1.69%, 250=4.32%, 500=0.46%, 750=0.05%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.16%, sys=0.42%, ctx=8791652, majf=0, minf=20621
  IO depths    : 1=0.1%, 2=101.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4696692/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=555071MB, aggrb=78943KB/s, minb=78943KB/s, maxb=78943KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=625897MB, aggrb=89016KB/s, minb=89016KB/s, maxb=89016KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=268672MB, aggrb=38210KB/s, minb=38210KB/s, maxb=38210KB/s, mint=7200168msec, maxt=7200168msec

Run status group 3 (all jobs):
  WRITE: io=293544MB, aggrb=41747KB/s, minb=41747KB/s, maxb=41747KB/s, mint=7200137msec, maxt=7200137msec



# 
# Wed 13 Jan 2016 09:07:08 PM EST
# 8kB randrw with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randrw with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6017 IOPS | 32 MBps | 7081 μsec Average Latency 
SATA-R5-1: 5822 IOPS | 31 MBps | 7363 μsec Average Latency 
NVMe-R5-0: 10247 IOPS | 54 MBps | 4079 μsec Average Latency 
NVMe-R5-1: 10547 IOPS | 56 MBps | 3983 μsec Average Latency 

Totals
IOPS: 32633
MBPS: 173
Average Latency (μsec): 5626

SATA-R5-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=172464: Wed Jan 13 23:08:09 2016
  read : io=226821MB, bw=32259KB/s, iops=4032, runt=7200013msec
    slat (usec): min=2, max=1929, avg=20.11, stdev=20.51
    clat (usec): min=19, max=1489.3K, avg=1865.15, stdev=4891.05
     lat (usec): min=146, max=1489.3K, avg=1885.47, stdev=4892.06
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  458], 10.00th=[  502], 20.00th=[  572],
     | 30.00th=[  620], 40.00th=[  668], 50.00th=[  724], 60.00th=[  804],
     | 70.00th=[  964], 80.00th=[ 1720], 90.00th=[ 3152], 95.00th=[ 9792],
     | 99.00th=[16768], 99.50th=[18304], 99.90th=[25216], 99.95th=[28288],
     | 99.99th=[78336]
    bw (KB  /s): min=    0, max=26048, per=25.29%, avg=8157.37, stdev=8613.19
  write: io=111708MB, bw=15887KB/s, iops=1985, runt=7200013msec
    slat (usec): min=2, max=1381, avg=25.45, stdev=23.28
    clat (usec): min=1, max=1511.6K, avg=12252.44, stdev=32368.87
     lat (usec): min=131, max=1511.6K, avg=12278.10, stdev=32369.99
    clat percentiles (usec):
     |  1.00th=[  386],  5.00th=[  604], 10.00th=[ 1352], 20.00th=[ 2608],
     | 30.00th=[ 3216], 40.00th=[ 3728], 50.00th=[ 4192], 60.00th=[ 4768],
     | 70.00th=[ 5536], 80.00th=[ 7200], 90.00th=[16768], 95.00th=[59648],
     | 99.00th=[181248], 99.50th=[207872], 99.90th=[259072], 99.95th=[301056],
     | 99.99th=[651264]
    bw (KB  /s): min=    0, max=12736, per=25.29%, avg=4017.62, stdev=4240.94
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.05%, 500=7.53%, 750=30.67%, 1000=12.44%
    lat (msec) : 2=8.56%, 4=17.74%, 10=15.37%, 20=4.34%, 50=1.48%
    lat (msec) : 100=0.73%, 250=1.05%, 500=0.03%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.60%, sys=1.70%, ctx=72960917, majf=0, minf=14712
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=29033032/w=14298544/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=172468: Wed Jan 13 23:08:09 2016
  read : io=219446MB, bw=31210KB/s, iops=3901, runt=7200009msec
    slat (usec): min=2, max=1618, avg=13.54, stdev=12.80
    clat (usec): min=27, max=1566.1K, avg=1849.06, stdev=4989.12
     lat (usec): min=159, max=1566.1K, avg=1862.73, stdev=4989.80
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  450], 10.00th=[  490], 20.00th=[  556],
     | 30.00th=[  604], 40.00th=[  644], 50.00th=[  700], 60.00th=[  772],
     | 70.00th=[  916], 80.00th=[ 1672], 90.00th=[ 3120], 95.00th=[ 9792],
     | 99.00th=[17280], 99.50th=[18816], 99.90th=[25472], 99.95th=[28032],
     | 99.99th=[54528]
    bw (KB  /s): min=    0, max=26176, per=25.29%, avg=7892.87, stdev=8462.97
  write: io=108088MB, bw=15372KB/s, iops=1921, runt=7200009msec
    slat (usec): min=2, max=964, avg=17.33, stdev=15.14
    clat (usec): min=21, max=1757.8K, avg=12848.47, stdev=33854.10
     lat (usec): min=138, max=1757.8K, avg=12865.93, stdev=33854.88
    clat percentiles (usec):
     |  1.00th=[  362],  5.00th=[  580], 10.00th=[ 1448], 20.00th=[ 2736],
     | 30.00th=[ 3344], 40.00th=[ 3824], 50.00th=[ 4320], 60.00th=[ 4896],
     | 70.00th=[ 5792], 80.00th=[ 7584], 90.00th=[17792], 95.00th=[63232],
     | 99.00th=[191488], 99.50th=[214016], 99.90th=[264192], 99.95th=[292864],
     | 99.99th=[651264]
    bw (KB  /s): min=    0, max=12864, per=25.29%, avg=3887.48, stdev=4167.12
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.05%, 500=8.93%, 750=32.00%
    lat (usec) : 1000=10.46%
    lat (msec) : 2=7.66%, 4=17.30%, 10=15.67%, 20=4.51%, 50=1.55%
    lat (msec) : 100=0.69%, 250=1.14%, 500=0.04%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.46%, sys=1.31%, ctx=71549545, majf=0, minf=14175
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=28089092/w=13835236/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=172472: Wed Jan 13 23:08:09 2016
  read : io=386190MB, bw=54925KB/s, iops=6865, runt=7200004msec
    slat (usec): min=2, max=3185, avg=21.50, stdev=21.08
    clat (usec): min=1, max=1623.5K, avg=1234.07, stdev=3661.09
     lat (usec): min=114, max=1623.5K, avg=1255.78, stdev=3662.03
    clat percentiles (usec):
     |  1.00th=[  334],  5.00th=[  418], 10.00th=[  462], 20.00th=[  516],
     | 30.00th=[  580], 40.00th=[  628], 50.00th=[  700], 60.00th=[  820],
     | 70.00th=[ 1176], 80.00th=[ 2008], 90.00th=[ 2736], 95.00th=[ 3472],
     | 99.00th=[ 4896], 99.50th=[ 5536], 99.90th=[ 7584], 99.95th=[ 8512],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=36055, per=25.13%, avg=13802.95, stdev=9573.64
  write: io=190252MB, bw=27058KB/s, iops=3382, runt=7200004msec
    slat (usec): min=3, max=3182, avg=27.08, stdev=23.53
    clat (usec): min=1, max=1633.4K, avg=6876.79, stdev=11130.41
     lat (usec): min=139, max=1633.6K, avg=6904.08, stdev=11132.92
    clat percentiles (usec):
     |  1.00th=[  366],  5.00th=[  548], 10.00th=[  884], 20.00th=[ 1736],
     | 30.00th=[ 2192], 40.00th=[ 2544], 50.00th=[ 2960], 60.00th=[ 3632],
     | 70.00th=[ 6432], 80.00th=[12608], 90.00th=[19328], 95.00th=[24192],
     | 99.00th=[32384], 99.50th=[35584], 99.90th=[42240], 99.95th=[47872],
     | 99.99th=[419840]
    bw (KB  /s): min=   13, max=17804, per=25.13%, avg=6800.72, stdev=4720.92
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.14%, 500=12.25%, 750=27.52%, 1000=8.36%
    lat (msec) : 2=13.74%, 4=23.79%, 10=6.22%, 20=4.92%, 50=3.05%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.16%, sys=3.19%, ctx=122122968, majf=0, minf=14503
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49432356/w=24352215/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=172476: Wed Jan 13 23:08:09 2016
  read : io=397517MB, bw=56536KB/s, iops=7066, runt=7200004msec
    slat (usec): min=2, max=819, avg=17.34, stdev=16.50
    clat (usec): min=1, max=1598.1K, avg=1165.67, stdev=3639.54
     lat (usec): min=105, max=1598.2K, avg=1183.18, stdev=3639.99
    clat percentiles (usec):
     |  1.00th=[  326],  5.00th=[  406], 10.00th=[  454], 20.00th=[  502],
     | 30.00th=[  564], 40.00th=[  612], 50.00th=[  676], 60.00th=[  788],
     | 70.00th=[ 1112], 80.00th=[ 1880], 90.00th=[ 2576], 95.00th=[ 3120],
     | 99.00th=[ 4576], 99.50th=[ 5088], 99.90th=[ 7200], 99.95th=[ 8160],
     | 99.99th=[11840]
    bw (KB  /s): min=    0, max=35552, per=25.15%, avg=14217.29, stdev=9145.05
  write: io=195829MB, bw=27851KB/s, iops=3481, runt=7200004msec
    slat (usec): min=2, max=603, avg=21.75, stdev=18.92
    clat (usec): min=18, max=1603.9K, avg=6761.55, stdev=11040.59
     lat (usec): min=137, max=1603.1K, avg=6783.48, stdev=11041.98
    clat percentiles (usec):
     |  1.00th=[  346],  5.00th=[  524], 10.00th=[ 1004], 20.00th=[ 1944],
     | 30.00th=[ 2416], 40.00th=[ 2800], 50.00th=[ 3248], 60.00th=[ 4016],
     | 70.00th=[ 7200], 80.00th=[12224], 90.00th=[17792], 95.00th=[21888],
     | 99.00th=[28800], 99.50th=[31360], 99.90th=[37632], 99.95th=[42752],
     | 99.99th=[436224]
    bw (KB  /s): min=    0, max=17872, per=25.15%, avg=7004.61, stdev=4500.52
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.16%
    lat (usec) : 500=14.43%, 750=26.73%, 1000=7.35%
    lat (msec) : 2=12.87%, 4=23.76%, 10=6.63%, 20=5.72%, 50=2.32%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.98%, sys=2.81%, ctx=128153830, majf=0, minf=14789
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=50882151/w=25066124/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=226821MB, aggrb=32258KB/s, minb=32258KB/s, maxb=32258KB/s, mint=7200013msec, maxt=7200013msec
  WRITE: io=111708MB, aggrb=15887KB/s, minb=15887KB/s, maxb=15887KB/s, mint=7200013msec, maxt=7200013msec

Run status group 1 (all jobs):
   READ: io=219446MB, aggrb=31210KB/s, minb=31210KB/s, maxb=31210KB/s, mint=7200009msec, maxt=7200009msec
  WRITE: io=108088MB, aggrb=15372KB/s, minb=15372KB/s, maxb=15372KB/s, mint=7200009msec, maxt=7200009msec

Run status group 2 (all jobs):
   READ: io=386190MB, aggrb=54924KB/s, minb=54924KB/s, maxb=54924KB/s, mint=7200004msec, maxt=7200004msec
  WRITE: io=190252MB, aggrb=27058KB/s, minb=27058KB/s, maxb=27058KB/s, mint=7200004msec, maxt=7200004msec

Run status group 3 (all jobs):
   READ: io=397517MB, aggrb=56535KB/s, minb=56535KB/s, maxb=56535KB/s, mint=7200004msec, maxt=7200004msec
  WRITE: io=195829MB, aggrb=27851KB/s, minb=27851KB/s, maxb=27851KB/s, mint=7200004msec, maxt=7200004msec



# 
# Wed 13 Jan 2016 11:08:12 PM EST
# 64kB randrw with SATA 5 jobs and 4 I/Os Outstanding
# 64kB randrw with NVMe 5 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1848 IOPS | 79 MBps | 14642 μsec Average Latency 
SATA-R5-1: 1820 IOPS | 78 MBps | 14824 μsec Average Latency 
NVMe-R5-0: 1532 IOPS | 65 MBps | 7780 μsec Average Latency 
NVMe-R5-1: 1622 IOPS | 69 MBps | 7344 μsec Average Latency 

Totals
IOPS: 6822
MBPS: 291
Average Latency (μsec): 11147

SATA-R5-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=177748: Thu Jan 14 01:09:12 2016
  read : io=557625MB, bw=79307KB/s, iops=1239, runt=7200010msec
    slat (usec): min=5, max=1039, avg=30.55, stdev=21.25
    clat (usec): min=40, max=4371.4K, avg=3373.17, stdev=30928.02
     lat (usec): min=249, max=4371.2K, avg=3403.92, stdev=30928.16
    clat percentiles (usec):
     |  1.00th=[  740],  5.00th=[  892], 10.00th=[  980], 20.00th=[ 1096],
     | 30.00th=[ 1176], 40.00th=[ 1272], 50.00th=[ 1368], 60.00th=[ 1480],
     | 70.00th=[ 1672], 80.00th=[ 2608], 90.00th=[ 5088], 95.00th=[ 7840],
     | 99.00th=[17536], 99.50th=[42752], 99.90th=[201728], 99.95th=[602112],
     | 99.99th=[1564672]
    bw (KB  /s): min=    1, max=52480, per=23.35%, avg=18519.76, stdev=12528.88
  write: io=274284MB, bw=39009KB/s, iops=609, runt=7200010msec
    slat (usec): min=8, max=993, avg=51.23, stdev=29.20
    clat (usec): min=16, max=4459.9K, avg=25830.55, stdev=101185.28
     lat (usec): min=251, max=4459.1K, avg=25882.01, stdev=101185.51
    clat percentiles (usec):
     |  1.00th=[  498],  5.00th=[ 1128], 10.00th=[ 4896], 20.00th=[ 7264],
     | 30.00th=[ 8512], 40.00th=[ 9408], 50.00th=[10304], 60.00th=[11456],
     | 70.00th=[13632], 80.00th=[18304], 90.00th=[47872], 95.00th=[71168],
     | 99.00th=[272384], 99.50th=[544768], 99.90th=[1630208], 99.95th=[2056192],
     | 99.99th=[2801664]
    bw (KB  /s): min=   19, max=25216, per=23.38%, avg=9120.26, stdev=6111.41
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.34%
    lat (usec) : 750=1.08%, 1000=7.56%
    lat (msec) : 2=44.66%, 4=7.64%, 10=19.04%, 20=13.10%, 50=3.15%
    lat (msec) : 100=2.13%, 250=0.88%, 500=0.20%, 750=0.06%, 1000=0.04%
    lat (msec) : 2000=0.09%, >=2000=0.02%
  cpu          : usr=0.32%, sys=0.83%, ctx=25533053, majf=0, minf=49610
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8922002/w=4388536/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=177753: Thu Jan 14 01:09:12 2016
  read : io=549213MB, bw=78110KB/s, iops=1220, runt=7200009msec
    slat (usec): min=5, max=1996, avg=40.93, stdev=28.43
    clat (usec): min=1, max=4150.9K, avg=3494.49, stdev=31699.18
     lat (usec): min=233, max=4150.9K, avg=3535.73, stdev=31699.24
    clat percentiles (usec):
     |  1.00th=[  740],  5.00th=[  892], 10.00th=[  972], 20.00th=[ 1096],
     | 30.00th=[ 1192], 40.00th=[ 1272], 50.00th=[ 1384], 60.00th=[ 1512],
     | 70.00th=[ 1752], 80.00th=[ 2928], 90.00th=[ 5472], 95.00th=[ 8096],
     | 99.00th=[16768], 99.50th=[42240], 99.90th=[203776], 99.95th=[643072],
     | 99.99th=[1613824]
    bw (KB  /s): min=    1, max=52736, per=23.27%, avg=18176.48, stdev=12529.47
  write: io=270134MB, bw=38419KB/s, iops=600, runt=7200009msec
    slat (usec): min=9, max=790, avg=64.79, stdev=36.07
    clat (usec): min=92, max=4234.4K, avg=26047.86, stdev=99834.35
     lat (usec): min=264, max=4234.4K, avg=26113.00, stdev=99834.22
    clat percentiles (usec):
     |  1.00th=[  510],  5.00th=[ 1080], 10.00th=[ 3920], 20.00th=[ 7008],
     | 30.00th=[ 8256], 40.00th=[ 9280], 50.00th=[10304], 60.00th=[11584],
     | 70.00th=[14144], 80.00th=[19840], 90.00th=[50432], 95.00th=[70144],
     | 99.00th=[272384], 99.50th=[536576], 99.90th=[1613824], 99.95th=[2007040],
     | 99.99th=[2801664]
    bw (KB  /s): min=    1, max=24064, per=23.28%, avg=8944.97, stdev=6117.18
    lat (usec) : 2=0.01%, 100=0.01%, 250=0.01%, 500=0.31%, 750=1.21%
    lat (usec) : 1000=7.75%
    lat (msec) : 2=43.29%, 4=8.27%, 10=19.63%, 20=12.41%, 50=3.53%
    lat (msec) : 100=2.35%, 250=0.82%, 500=0.20%, 750=0.06%, 1000=0.04%
    lat (msec) : 2000=0.09%, >=2000=0.02%
  cpu          : usr=0.45%, sys=0.97%, ctx=24915296, majf=0, minf=49395
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8787401/w=4322133/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=177758: Thu Jan 14 01:09:12 2016
  read : io=462299MB, bw=65749KB/s, iops=1027, runt=7200041msec
    slat (usec): min=6, max=813, avg=43.58, stdev=29.66
    clat (usec): min=1, max=4136.4K, avg=4017.45, stdev=35928.14
     lat (usec): min=203, max=4136.4K, avg=4061.36, stdev=35928.18
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  724], 10.00th=[  828], 20.00th=[ 1012],
     | 30.00th=[ 1272], 40.00th=[ 1880], 50.00th=[ 2576], 60.00th=[ 3056],
     | 70.00th=[ 3632], 80.00th=[ 4320], 90.00th=[ 5152], 95.00th=[ 5920],
     | 99.00th=[ 8384], 99.50th=[11072], 99.90th=[284672], 99.95th=[806912],
     | 99.99th=[1777664]
    bw (KB  /s): min=    1, max=66688, per=23.50%, avg=15448.36, stdev=10220.41
  write: io=227689MB, bw=32382KB/s, iops=505, runt=7200041msec
    slat (usec): min=9, max=749, avg=67.86, stdev=38.15
    clat (usec): min=1, max=4155.9K, avg=11432.59, stdev=66971.25
     lat (usec): min=255, max=4155.9K, avg=11500.81, stdev=66971.06
    clat percentiles (usec):
     |  1.00th=[  450],  5.00th=[  756], 10.00th=[  996], 20.00th=[ 1608],
     | 30.00th=[ 2448], 40.00th=[ 4080], 50.00th=[ 6560], 60.00th=[ 8768],
     | 70.00th=[10816], 80.00th=[12864], 90.00th=[15296], 95.00th=[17280],
     | 99.00th=[88576], 99.50th=[166912], 99.90th=[1187840], 99.95th=[1581056],
     | 99.99th=[2408448]
    bw (KB  /s): min=    1, max=32896, per=23.51%, avg=7613.20, stdev=5033.83
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.02%, 500=0.80%, 750=5.01%, 1000=10.55%
    lat (msec) : 2=19.80%, 4=27.59%, 10=24.50%, 20=10.70%, 50=0.40%
    lat (msec) : 100=0.10%, 250=0.33%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.06%, >=2000=0.01%
  cpu          : usr=0.41%, sys=0.86%, ctx=21389738, majf=0, minf=28538
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7396787/w=3643022/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=177763: Thu Jan 14 01:09:12 2016
  read : io=489448MB, bw=69610KB/s, iops=1087, runt=7200006msec
    slat (usec): min=6, max=1015, avg=38.40, stdev=27.00
    clat (usec): min=1, max=4154.9K, avg=3816.62, stdev=35656.33
     lat (usec): min=192, max=4154.9K, avg=3855.31, stdev=35656.36
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  724], 10.00th=[  828], 20.00th=[ 1020],
     | 30.00th=[ 1256], 40.00th=[ 1752], 50.00th=[ 2416], 60.00th=[ 2864],
     | 70.00th=[ 3344], 80.00th=[ 4016], 90.00th=[ 4832], 95.00th=[ 5472],
     | 99.00th=[ 7968], 99.50th=[10432], 99.90th=[259072], 99.95th=[782336],
     | 99.99th=[1777664]
    bw (KB  /s): min=    1, max=67328, per=23.51%, avg=16364.22, stdev=10243.65
  write: io=240885MB, bw=34259KB/s, iops=535, runt=7200006msec
    slat (usec): min=8, max=742, avg=61.13, stdev=35.22
    clat (usec): min=1, max=4155.4K, avg=10771.67, stdev=64951.11
     lat (usec): min=230, max=4155.4K, avg=10833.13, stdev=64951.03
    clat percentiles (usec):
     |  1.00th=[  442],  5.00th=[  756], 10.00th=[ 1020], 20.00th=[ 1672],
     | 30.00th=[ 2512], 40.00th=[ 3984], 50.00th=[ 6176], 60.00th=[ 8256],
     | 70.00th=[10048], 80.00th=[11840], 90.00th=[14144], 95.00th=[16064],
     | 99.00th=[78336], 99.50th=[146432], 99.90th=[1122304], 99.95th=[1548288],
     | 99.99th=[2473984]
    bw (KB  /s): min=    1, max=34048, per=23.52%, avg=8057.10, stdev=5028.85
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.02%, 500=0.82%, 750=4.83%, 1000=10.36%
    lat (msec) : 2=21.18%, 4=29.61%, 10=22.86%, 20=9.49%, 50=0.22%
    lat (msec) : 100=0.10%, 250=0.33%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.05%, >=2000=0.01%
  cpu          : usr=0.37%, sys=0.84%, ctx=22708724, majf=0, minf=29502
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7831160/w=3854158/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=557625MB, aggrb=79306KB/s, minb=79306KB/s, maxb=79306KB/s, mint=7200010msec, maxt=7200010msec
  WRITE: io=274284MB, aggrb=39009KB/s, minb=39009KB/s, maxb=39009KB/s, mint=7200010msec, maxt=7200010msec

Run status group 1 (all jobs):
   READ: io=549213MB, aggrb=78110KB/s, minb=78110KB/s, maxb=78110KB/s, mint=7200009msec, maxt=7200009msec
  WRITE: io=270134MB, aggrb=38419KB/s, minb=38419KB/s, maxb=38419KB/s, mint=7200009msec, maxt=7200009msec

Run status group 2 (all jobs):
   READ: io=462299MB, aggrb=65748KB/s, minb=65748KB/s, maxb=65748KB/s, mint=7200041msec, maxt=7200041msec
  WRITE: io=227689MB, aggrb=32382KB/s, minb=32382KB/s, maxb=32382KB/s, mint=7200041msec, maxt=7200041msec

Run status group 3 (all jobs):
   READ: io=489448MB, aggrb=69610KB/s, minb=69610KB/s, maxb=69610KB/s, mint=7200006msec, maxt=7200006msec
  WRITE: io=240885MB, aggrb=34259KB/s, minb=34259KB/s, maxb=34259KB/s, mint=7200006msec, maxt=7200006msec



# 
# Thu 14 Jan 2016 01:09:15 AM EST
# 1mB write with SATA 1 jobs and 2 I/Os Outstanding
# 1mB write with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 301 IOPS | 308 MBps | 6630 μsec Average Latency 
SATA-R5-1: 280 IOPS | 287 MBps | 7120 μsec Average Latency 
NVMe-R5-0: 133 IOPS | 136 MBps | 60040 μsec Average Latency 
NVMe-R5-1: 130 IOPS | 133 MBps | 61470 μsec Average Latency 

Totals
IOPS: 844
MBPS: 864
Average Latency (μsec): 33815

SATA-R5-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
SATA-R5-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-01: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=183730: Thu Jan 14 03:10:16 2016
  write: io=2119.1GB, bw=308743KB/s, iops=301, runt=7200002msec
    slat (usec): min=102, max=1993, avg=208.26, stdev=47.95
    clat (usec): min=861, max=555529, avg=6422.32, stdev=16671.61
     lat (msec): min=1, max=555, avg= 6.63, stdev=16.67
    clat percentiles (usec):
     |  1.00th=[ 1384],  5.00th=[ 1624], 10.00th=[ 1816], 20.00th=[ 1912],
     | 30.00th=[ 1976], 40.00th=[ 2040], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2416], 80.00th=[ 3056], 90.00th=[ 7520], 95.00th=[41728],
     | 99.00th=[80384], 99.50th=[104960], 99.90th=[166912], 99.95th=[207872],
     | 99.99th=[305152]
    bw (KB  /s): min=   17, max=774642, per=100.00%, avg=310430.85, stdev=98716.61
    lat (usec) : 1000=0.01%
    lat (msec) : 2=33.01%, 4=51.41%, 10=8.20%, 20=0.85%, 50=2.87%
    lat (msec) : 100=3.08%, 250=0.54%, 500=0.03%, 750=0.01%
  cpu          : usr=1.35%, sys=2.90%, ctx=10392014, majf=0, minf=66720
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2170846/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=183731: Thu Jan 14 03:10:16 2016
  write: io=1974.5GB, bw=287544KB/s, iops=280, runt=7200002msec
    slat (usec): min=100, max=1584, avg=202.99, stdev=46.38
    clat (usec): min=941, max=606639, avg=6916.64, stdev=17837.63
     lat (msec): min=1, max=606, avg= 7.12, stdev=17.84
    clat percentiles (usec):
     |  1.00th=[ 1368],  5.00th=[ 1608], 10.00th=[ 1816], 20.00th=[ 1912],
     | 30.00th=[ 1976], 40.00th=[ 2040], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2448], 80.00th=[ 3152], 90.00th=[ 7776], 95.00th=[45312],
     | 99.00th=[85504], 99.50th=[109056], 99.90th=[177152], 99.95th=[211968],
     | 99.99th=[321536]
    bw (KB  /s): min=   17, max=764506, per=100.00%, avg=289218.86, stdev=96296.82
    lat (usec) : 1000=0.01%
    lat (msec) : 2=32.78%, 4=50.84%, 10=8.31%, 20=0.91%, 50=2.88%
    lat (msec) : 100=3.64%, 250=0.61%, 500=0.03%, 750=0.01%
  cpu          : usr=1.27%, sys=2.59%, ctx=9704204, majf=0, minf=67242
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2021792/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=183732: Thu Jan 14 03:10:16 2016
  write: io=959331MB, bw=136436KB/s, iops=133, runt=7200112msec
    slat (usec): min=101, max=1510, avg=313.81, stdev=84.05
    clat (msec): min=1, max=1363, avg=59.73, stdev=75.22
     lat (msec): min=1, max=1363, avg=60.04, stdev=75.21
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   13], 60.00th=[   60],
     | 70.00th=[   80], 80.00th=[  114], 90.00th=[  161], 95.00th=[  206],
     | 99.00th=[  306], 99.50th=[  396], 99.90th=[  603], 99.95th=[  652],
     | 99.99th=[  889]
    bw (KB  /s): min=   17, max=589513, per=100.00%, avg=137830.91, stdev=52346.13
    lat (msec) : 2=0.52%, 4=7.20%, 10=39.82%, 20=5.00%, 50=4.12%
    lat (msec) : 100=20.29%, 250=20.94%, 500=1.96%, 750=0.13%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.79%, sys=1.74%, ctx=4379546, majf=0, minf=68811
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=959324/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=183733: Thu Jan 14 03:10:16 2016
  write: io=936998MB, bw=133262KB/s, iops=130, runt=7200004msec
    slat (usec): min=103, max=1249, avg=295.85, stdev=89.13
    clat (msec): min=1, max=1303, avg=61.18, stdev=76.24
     lat (msec): min=1, max=1303, avg=61.47, stdev=76.23
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   14], 60.00th=[   61],
     | 70.00th=[   82], 80.00th=[  117], 90.00th=[  165], 95.00th=[  206],
     | 99.00th=[  314], 99.50th=[  392], 99.90th=[  553], 99.95th=[  644],
     | 99.99th=[  848]
    bw (KB  /s): min=   17, max=548283, per=100.00%, avg=134511.91, stdev=50056.09
    lat (msec) : 2=0.50%, 4=7.75%, 10=38.39%, 20=5.33%, 50=4.20%
    lat (msec) : 100=20.01%, 250=21.50%, 500=2.17%, 750=0.12%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.77%, sys=1.69%, ctx=4300062, majf=0, minf=63561
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=936991/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
  WRITE: io=2119.1GB, aggrb=308742KB/s, minb=308742KB/s, maxb=308742KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
  WRITE: io=1974.5GB, aggrb=287543KB/s, minb=287543KB/s, maxb=287543KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=959331MB, aggrb=136436KB/s, minb=136436KB/s, maxb=136436KB/s, mint=7200112msec, maxt=7200112msec

Run status group 3 (all jobs):
  WRITE: io=936998MB, aggrb=133261KB/s, minb=133261KB/s, maxb=133261KB/s, mint=7200004msec, maxt=7200004msec



# 
# Thu 14 Jan 2016 03:10:18 AM EST
# 1mB read with SATA 1 jobs and 8 I/Os Outstanding
# 1mB read with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 738 IOPS | 755 MBps | 10840 μsec Average Latency 
SATA-R5-1: 739 IOPS | 757 MBps | 10810 μsec Average Latency 
NVMe-R5-0: 372 IOPS | 381 MBps | 5370 μsec Average Latency 
NVMe-R5-1: 357 IOPS | 365 MBps | 5590 μsec Average Latency 

Totals
IOPS: 2206
MBPS: 2258
Average Latency (μsec): 8152

SATA-R5-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=188937: Thu Jan 14 05:11:18 2016
  read : io=5189.1GB, bw=755839KB/s, iops=738, runt=7200011msec
    slat (usec): min=72, max=10324, avg=213.59, stdev=131.10
    clat (msec): min=1, max=53, avg=10.62, stdev= 2.51
     lat (msec): min=1, max=53, avg=10.84, stdev= 2.50
    clat percentiles (usec):
     |  1.00th=[ 5472],  5.00th=[ 7264], 10.00th=[ 8160], 20.00th=[ 9024],
     | 30.00th=[ 9408], 40.00th=[ 9792], 50.00th=[10048], 60.00th=[10432],
     | 70.00th=[11072], 80.00th=[12224], 90.00th=[14144], 95.00th=[15808],
     | 99.00th=[18304], 99.50th=[19328], 99.90th=[22144], 99.95th=[23680],
     | 99.99th=[26496]
    bw (KB  /s): min=   17, max=848573, per=100.00%, avg=756475.21, stdev=33450.37
    lat (msec) : 2=0.01%, 4=0.08%, 10=47.47%, 20=52.13%, 50=0.31%
    lat (msec) : 100=0.01%
  cpu          : usr=0.36%, sys=9.32%, ctx=26032962, majf=0, minf=68801
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5314493/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=188938: Thu Jan 14 05:11:18 2016
  read : io=5200.3GB, bw=757304KB/s, iops=739, runt=7200008msec
    slat (usec): min=71, max=9546, avg=220.39, stdev=138.82
    clat (msec): min=1, max=38, avg=10.59, stdev= 2.55
     lat (msec): min=1, max=38, avg=10.81, stdev= 2.55
    clat percentiles (usec):
     |  1.00th=[ 5344],  5.00th=[ 7072], 10.00th=[ 8032], 20.00th=[ 8896],
     | 30.00th=[ 9408], 40.00th=[ 9792], 50.00th=[10048], 60.00th=[10432],
     | 70.00th=[11072], 80.00th=[12224], 90.00th=[14272], 95.00th=[15808],
     | 99.00th=[18304], 99.50th=[19328], 99.90th=[22144], 99.95th=[23424],
     | 99.99th=[26496]
    bw (KB  /s): min=   17, max=854016, per=100.00%, avg=757938.15, stdev=31219.99
    lat (msec) : 2=0.01%, 4=0.09%, 10=48.04%, 20=51.55%, 50=0.32%
  cpu          : usr=0.36%, sys=9.42%, ctx=26154579, majf=0, minf=70906
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5324796/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=188939: Thu Jan 14 05:11:18 2016
  read : io=2617.2GB, bw=381148KB/s, iops=372, runt=7200007msec
    slat (usec): min=79, max=6891, avg=250.12, stdev=102.62
    clat (usec): min=473, max=51820, avg=5117.24, stdev=2422.07
     lat (msec): min=1, max=52, avg= 5.37, stdev= 2.43
    clat percentiles (usec):
     |  1.00th=[ 1432],  5.00th=[ 1896], 10.00th=[ 2448], 20.00th=[ 3440],
     | 30.00th=[ 4016], 40.00th=[ 4384], 50.00th=[ 4704], 60.00th=[ 5024],
     | 70.00th=[ 5408], 80.00th=[ 6176], 90.00th=[ 8768], 95.00th=[10432],
     | 99.00th=[12864], 99.50th=[13632], 99.90th=[15936], 99.95th=[17024],
     | 99.99th=[19584]
    bw (KB  /s): min=   17, max=457836, per=100.00%, avg=381497.18, stdev=25153.20
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=5.95%, 4=23.60%, 10=64.09%, 20=6.35%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.32%, sys=4.44%, ctx=13391042, majf=0, minf=67779
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2679945/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=188940: Thu Jan 14 05:11:18 2016
  read : io=2511.5GB, bw=365748KB/s, iops=357, runt=7200009msec
    slat (usec): min=81, max=6941, avg=244.29, stdev=103.39
    clat (usec): min=600, max=30015, avg=5349.45, stdev=2457.77
     lat (msec): min=1, max=30, avg= 5.59, stdev= 2.46
    clat percentiles (usec):
     |  1.00th=[ 1512],  5.00th=[ 2128], 10.00th=[ 2864], 20.00th=[ 3760],
     | 30.00th=[ 4192], 40.00th=[ 4512], 50.00th=[ 4768], 60.00th=[ 5088],
     | 70.00th=[ 5536], 80.00th=[ 6496], 90.00th=[ 9152], 95.00th=[10816],
     | 99.00th=[13120], 99.50th=[14016], 99.90th=[16320], 99.95th=[17536],
     | 99.99th=[20096]
    bw (KB  /s): min=   17, max=447616, per=100.00%, avg=366098.35, stdev=26840.99
    lat (usec) : 750=0.01%, 1000=0.01%
    lat (msec) : 2=4.15%, 4=20.74%, 10=67.79%, 20=7.31%, 50=0.01%
  cpu          : usr=0.30%, sys=4.06%, ctx=12878707, majf=0, minf=67382
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2571669/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=5189.1GB, aggrb=755838KB/s, minb=755838KB/s, maxb=755838KB/s, mint=7200011msec, maxt=7200011msec

Run status group 1 (all jobs):
   READ: io=5200.3GB, aggrb=757304KB/s, minb=757304KB/s, maxb=757304KB/s, mint=7200008msec, maxt=7200008msec

Run status group 2 (all jobs):
   READ: io=2617.2GB, aggrb=381147KB/s, minb=381147KB/s, maxb=381147KB/s, mint=7200007msec, maxt=7200007msec

Run status group 3 (all jobs):
   READ: io=2511.5GB, aggrb=365748KB/s, minb=365748KB/s, maxb=365748KB/s, mint=7200009msec, maxt=7200009msec



# 
# Thu 14 Jan 2016 05:11:21 AM EST
# 256kB write with SATA 1 jobs and 8 I/Os Outstanding
# 256kB write with NVMe 1 jobs and 1 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1478 IOPS | 378 MBps | 5407 μsec Average Latency 
SATA-R5-1: 1448 IOPS | 370 MBps | 5522 μsec Average Latency 
NVMe-R5-0: 512 IOPS | 131 MBps | 1946 μsec Average Latency 
NVMe-R5-1: 498 IOPS | 127 MBps | 2002 μsec Average Latency 

Totals
IOPS: 3936
MBPS: 1006
Average Latency (μsec): 3719

SATA-R5-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
NVMe-R5-11: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=194054: Thu Jan 14 07:12:22 2016
  write: io=2599.5GB, bw=378511KB/s, iops=1478, runt=7200049msec
    slat (usec): min=26, max=632, avg=81.59, stdev=26.57
    clat (usec): min=367, max=470832, avg=5326.01, stdev=13800.08
     lat (usec): min=554, max=470900, avg=5407.82, stdev=13799.63
    clat percentiles (usec):
     |  1.00th=[  756],  5.00th=[  996], 10.00th=[ 1176], 20.00th=[ 1448],
     | 30.00th=[ 1576], 40.00th=[ 1656], 50.00th=[ 1752], 60.00th=[ 1816],
     | 70.00th=[ 1912], 80.00th=[ 2064], 90.00th=[ 3824], 95.00th=[37120],
     | 99.00th=[66048], 99.50th=[90624], 99.90th=[127488], 99.95th=[140288],
     | 99.99th=[175104]
    bw (KB  /s): min=    4, max=721408, per=100.00%, avg=379387.12, stdev=84555.94
    lat (usec) : 500=0.01%, 750=0.90%, 1000=4.25%
    lat (msec) : 2=72.31%, 4=12.78%, 10=2.02%, 20=0.27%, 50=5.15%
    lat (msec) : 100=1.97%, 250=0.35%, 500=0.01%
  cpu          : usr=3.03%, sys=5.28%, ctx=18863304, majf=0, minf=65282
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10645679/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=194055: Thu Jan 14 07:12:22 2016
  write: io=2545.7GB, bw=370729KB/s, iops=1448, runt=7200025msec
    slat (usec): min=26, max=1062, avg=53.60, stdev=14.38
    clat (usec): min=384, max=455706, avg=5468.73, stdev=14326.62
     lat (usec): min=529, max=455753, avg=5522.54, stdev=14326.21
    clat percentiles (usec):
     |  1.00th=[  748],  5.00th=[ 1004], 10.00th=[ 1208], 20.00th=[ 1480],
     | 30.00th=[ 1592], 40.00th=[ 1704], 50.00th=[ 1784], 60.00th=[ 1832],
     | 70.00th=[ 1928], 80.00th=[ 2064], 90.00th=[ 3696], 95.00th=[39680],
     | 99.00th=[68096], 99.50th=[92672], 99.90th=[132096], 99.95th=[144384],
     | 99.99th=[183296]
    bw (KB  /s): min=    4, max=792576, per=100.00%, avg=371601.48, stdev=86119.98
    lat (usec) : 500=0.01%, 750=1.01%, 1000=3.97%
    lat (msec) : 2=72.29%, 4=13.21%, 10=2.01%, 20=0.25%, 50=4.51%
    lat (msec) : 100=2.37%, 250=0.38%, 500=0.01%
  cpu          : usr=1.85%, sys=4.26%, ctx=19754333, majf=0, minf=67371
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10426774/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=194056: Thu Jan 14 07:12:22 2016
  write: io=921882MB, bw=131112KB/s, iops=512, runt=7200001msec
    slat (usec): min=31, max=1369, avg=98.29, stdev=19.91
    clat (usec): min=53, max=400326, avg=1847.63, stdev=7927.47
     lat (usec): min=502, max=400411, avg=1946.34, stdev=7926.68
    clat percentiles (usec):
     |  1.00th=[  516],  5.00th=[  532], 10.00th=[  548], 20.00th=[  564],
     | 30.00th=[  580], 40.00th=[  596], 50.00th=[  620], 60.00th=[  652],
     | 70.00th=[  780], 80.00th=[ 1512], 90.00th=[ 1848], 95.00th=[ 2024],
     | 99.00th=[45824], 99.50th=[64256], 99.90th=[101888], 99.95th=[116224],
     | 99.99th=[148480]
    bw (KB  /s): min=    4, max=291840, per=100.00%, avg=131476.29, stdev=33765.34
    lat (usec) : 100=0.01%, 500=0.28%, 750=68.91%, 1000=4.33%
    lat (msec) : 2=21.16%, 4=3.26%, 10=0.33%, 20=0.06%, 50=0.79%
    lat (msec) : 100=0.76%, 250=0.11%, 500=0.01%
  cpu          : usr=1.27%, sys=2.23%, ctx=7429637, majf=0, minf=9308
  IO depths    : 1=100.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3687529/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=194057: Thu Jan 14 07:12:22 2016
  write: io=896481MB, bw=127499KB/s, iops=498, runt=7200001msec
    slat (usec): min=31, max=529, avg=101.94, stdev=18.87
    clat (usec): min=338, max=445616, avg=1899.59, stdev=8191.41
     lat (usec): min=515, max=445713, avg=2002.00, stdev=8190.85
    clat percentiles (usec):
     |  1.00th=[  510],  5.00th=[  532], 10.00th=[  548], 20.00th=[  564],
     | 30.00th=[  580], 40.00th=[  596], 50.00th=[  612], 60.00th=[  652],
     | 70.00th=[  748], 80.00th=[ 1528], 90.00th=[ 1848], 95.00th=[ 2040],
     | 99.00th=[47872], 99.50th=[66048], 99.90th=[102912], 99.95th=[118272],
     | 99.99th=[152576]
    bw (KB  /s): min=    4, max=290304, per=100.00%, avg=127894.58, stdev=33564.16
    lat (usec) : 500=0.29%, 750=69.64%, 1000=4.20%
    lat (msec) : 2=20.44%, 4=3.29%, 10=0.33%, 20=0.06%, 50=0.80%
    lat (msec) : 100=0.82%, 250=0.12%, 500=0.01%
  cpu          : usr=1.16%, sys=2.27%, ctx=7228819, majf=0, minf=9434
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3585923/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=2599.5GB, aggrb=378510KB/s, minb=378510KB/s, maxb=378510KB/s, mint=7200049msec, maxt=7200049msec

Run status group 1 (all jobs):
  WRITE: io=2545.7GB, aggrb=370728KB/s, minb=370728KB/s, maxb=370728KB/s, mint=7200025msec, maxt=7200025msec

Run status group 2 (all jobs):
  WRITE: io=921882MB, aggrb=131112KB/s, minb=131112KB/s, maxb=131112KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=896481MB, aggrb=127499KB/s, minb=127499KB/s, maxb=127499KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 07:12:24 AM EST
# 256kB read with SATA 1 jobs and 8 I/Os Outstanding
# 256kB read with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 2393 IOPS | 612 MBps | 3340 μsec Average Latency 
SATA-R5-1: 2396 IOPS | 613 MBps | 3336 μsec Average Latency 
NVMe-R5-0: 2140 IOPS | 547 MBps | 1866 μsec Average Latency 
NVMe-R5-1: 2128 IOPS | 544 MBps | 1876 μsec Average Latency 

Totals
IOPS: 9057
MBPS: 2316
Average Latency (μsec): 2604

SATA-R5-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=199210: Thu Jan 14 09:13:25 2016
  read : io=4207.3GB, bw=612718KB/s, iops=2393, runt=7200004msec
    slat (usec): min=17, max=2727, avg=64.80, stdev=49.36
    clat (usec): min=0, max=24059, avg=3275.37, stdev=917.93
     lat (usec): min=952, max=24159, avg=3340.39, stdev=915.99
    clat percentiles (usec):
     |  1.00th=[ 1960],  5.00th=[ 2384], 10.00th=[ 2544], 20.00th=[ 2736],
     | 30.00th=[ 2864], 40.00th=[ 2992], 50.00th=[ 3088], 60.00th=[ 3216],
     | 70.00th=[ 3376], 80.00th=[ 3568], 90.00th=[ 4128], 95.00th=[ 4832],
     | 99.00th=[ 7136], 99.50th=[ 8096], 99.90th=[10560], 99.95th=[11840],
     | 99.99th=[14784]
    bw (KB  /s): min=    4, max=658432, per=100.00%, avg=613035.78, stdev=16640.89
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=1.18%, 4=87.71%, 10=10.97%, 20=0.14%, 50=0.01%
  cpu          : usr=0.90%, sys=8.76%, ctx=32098262, majf=0, minf=67884
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=17232708/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=199211: Thu Jan 14 09:13:25 2016
  read : io=4211.9GB, bw=613388KB/s, iops=2396, runt=7200004msec
    slat (usec): min=17, max=2641, avg=64.57, stdev=48.61
    clat (usec): min=0, max=24927, avg=3271.98, stdev=937.89
     lat (usec): min=928, max=25073, avg=3336.76, stdev=936.04
    clat percentiles (usec):
     |  1.00th=[ 1912],  5.00th=[ 2320], 10.00th=[ 2512], 20.00th=[ 2736],
     | 30.00th=[ 2864], 40.00th=[ 2960], 50.00th=[ 3088], 60.00th=[ 3216],
     | 70.00th=[ 3376], 80.00th=[ 3600], 90.00th=[ 4128], 95.00th=[ 4896],
     | 99.00th=[ 7200], 99.50th=[ 8160], 99.90th=[10560], 99.95th=[11840],
     | 99.99th=[14784]
    bw (KB  /s): min=    4, max=662528, per=100.00%, avg=613706.48, stdev=16445.53
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=1.44%, 4=87.12%, 10=11.29%, 20=0.14%, 50=0.01%
  cpu          : usr=0.90%, sys=8.74%, ctx=32117393, majf=0, minf=68407
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=17251533/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=199212: Thu Jan 14 09:13:25 2016
  read : io=3762.8GB, bw=547994KB/s, iops=2140, runt=7200003msec
    slat (usec): min=17, max=2740, avg=68.43, stdev=53.56
    clat (usec): min=0, max=27248, avg=1797.73, stdev=689.58
     lat (usec): min=369, max=27315, avg=1866.40, stdev=687.49
    clat percentiles (usec):
     |  1.00th=[  724],  5.00th=[ 1032], 10.00th=[ 1208], 20.00th=[ 1384],
     | 30.00th=[ 1512], 40.00th=[ 1608], 50.00th=[ 1704], 60.00th=[ 1816],
     | 70.00th=[ 1944], 80.00th=[ 2096], 90.00th=[ 2352], 95.00th=[ 2704],
     | 99.00th=[ 4704], 99.50th=[ 5856], 99.90th=[ 8256], 99.95th=[ 9408],
     | 99.99th=[12096]
    bw (KB  /s): min=    4, max=610304, per=100.00%, avg=548299.00, stdev=16203.35
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.15%, 750=1.00%, 1000=3.04%
    lat (msec) : 2=70.17%, 4=24.29%, 10=1.30%, 20=0.03%, 50=0.01%
  cpu          : usr=0.85%, sys=8.15%, ctx=28971441, majf=0, minf=35832
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=15412341/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=199213: Thu Jan 14 09:13:25 2016
  read : io=3741.9GB, bw=544945KB/s, iops=2128, runt=7200002msec
    slat (usec): min=17, max=2730, avg=69.90, stdev=54.61
    clat (usec): min=0, max=21497, avg=1806.70, stdev=704.83
     lat (usec): min=362, max=21532, avg=1876.83, stdev=702.58
    clat percentiles (usec):
     |  1.00th=[  716],  5.00th=[ 1032], 10.00th=[ 1192], 20.00th=[ 1384],
     | 30.00th=[ 1512], 40.00th=[ 1608], 50.00th=[ 1720], 60.00th=[ 1832],
     | 70.00th=[ 1960], 80.00th=[ 2128], 90.00th=[ 2384], 95.00th=[ 2736],
     | 99.00th=[ 4704], 99.50th=[ 5920], 99.90th=[ 8256], 99.95th=[ 9408],
     | 99.99th=[12352]
    bw (KB  /s): min=    4, max=602624, per=100.00%, avg=545258.15, stdev=16701.72
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.17%, 750=1.06%, 1000=3.15%
    lat (msec) : 2=68.34%, 4=25.89%, 10=1.33%, 20=0.04%, 50=0.01%
  cpu          : usr=0.86%, sys=8.26%, ctx=28753984, majf=0, minf=33572
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=15326567/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=4207.3GB, aggrb=612718KB/s, minb=612718KB/s, maxb=612718KB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=4211.9GB, aggrb=613387KB/s, minb=613387KB/s, maxb=613387KB/s, mint=7200004msec, maxt=7200004msec

Run status group 2 (all jobs):
   READ: io=3762.8GB, aggrb=547994KB/s, minb=547994KB/s, maxb=547994KB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=3741.9GB, aggrb=544944KB/s, minb=544944KB/s, maxb=544944KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:13:28 AM EST
# 64kB write with SATA 1 jobs and 4 I/Os Outstanding
# 64kB write with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 5394 IOPS | 345 MBps | 739 μsec Average Latency 
SATA-R5-1: 6353 IOPS | 406 MBps | 628 μsec Average Latency 
NVMe-R5-0: 1906 IOPS | 122 MBps | 2096 μsec Average Latency 
NVMe-R5-1: 1879 IOPS | 120 MBps | 2126 μsec Average Latency 

Totals
IOPS: 15532
MBPS: 993
Average Latency (μsec): 1397

SATA-R5-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-01: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=204333: Thu Jan 14 11:14:28 2016
  write: io=2370.8GB, bw=345264KB/s, iops=5394, runt=7200001msec
    slat (usec): min=9, max=1587, avg=40.48, stdev=19.51
    clat (usec): min=50, max=411532, avg=699.01, stdev=4587.28
     lat (usec): min=203, max=411557, avg=739.67, stdev=4587.15
    clat percentiles (usec):
     |  1.00th=[  282],  5.00th=[  310], 10.00th=[  330], 20.00th=[  362],
     | 30.00th=[  394], 40.00th=[  422], 50.00th=[  442], 60.00th=[  462],
     | 70.00th=[  482], 80.00th=[  516], 90.00th=[  588], 95.00th=[  676],
     | 99.00th=[ 1480], 99.50th=[ 5792], 99.90th=[79360], 99.95th=[112128],
     | 99.99th=[199680]
    bw (KB  /s): min=    1, max=554368, per=100.00%, avg=346878.44, stdev=86411.45
    lat (usec) : 100=0.01%, 250=0.25%, 500=75.60%, 750=20.88%, 1000=1.59%
    lat (msec) : 2=0.94%, 4=0.21%, 10=0.09%, 20=0.05%, 50=0.25%
    lat (msec) : 100=0.08%, 250=0.07%, 500=0.01%
  cpu          : usr=2.95%, sys=9.74%, ctx=64417518, majf=0, minf=9525
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=38842180/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=204334: Thu Jan 14 11:14:28 2016
  write: io=2792.3GB, bw=406656KB/s, iops=6353, runt=7200001msec
    slat (usec): min=7, max=464, avg=25.95, stdev= 7.56
    clat (usec): min=122, max=368544, avg=602.17, stdev=3374.00
     lat (usec): min=194, max=368559, avg=628.31, stdev=3373.97
    clat percentiles (usec):
     |  1.00th=[  286],  5.00th=[  322], 10.00th=[  338], 20.00th=[  350],
     | 30.00th=[  378], 40.00th=[  406], 50.00th=[  438], 60.00th=[  466],
     | 70.00th=[  478], 80.00th=[  502], 90.00th=[  580], 95.00th=[  668],
     | 99.00th=[ 1304], 99.50th=[ 2320], 99.90th=[44288], 99.95th=[89600],
     | 99.99th=[132096]
    bw (KB  /s): min=    1, max=571392, per=100.00%, avg=408890.79, stdev=121585.41
    lat (usec) : 250=0.27%, 500=79.25%, 750=17.26%, 1000=1.64%
    lat (msec) : 2=1.00%, 4=0.21%, 10=0.08%, 20=0.03%, 50=0.16%
    lat (msec) : 100=0.05%, 250=0.04%, 500=0.01%
  cpu          : usr=2.64%, sys=7.55%, ctx=78417565, majf=0, minf=9512
  IO depths    : 1=0.1%, 2=0.1%, 4=100.6%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=45748798/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=204335: Thu Jan 14 11:14:28 2016
  write: io=857841MB, bw=122004KB/s, iops=1906, runt=7200001msec
    slat (usec): min=8, max=588, avg=38.27, stdev=23.47
    clat (usec): min=92, max=539077, avg=2057.98, stdev=13552.48
     lat (usec): min=208, max=539096, avg=2096.44, stdev=13552.37
    clat percentiles (usec):
     |  1.00th=[  282],  5.00th=[  330], 10.00th=[  358], 20.00th=[  406],
     | 30.00th=[  438], 40.00th=[  462], 50.00th=[  482], 60.00th=[  510],
     | 70.00th=[  548], 80.00th=[  596], 90.00th=[  684], 95.00th=[  852],
     | 99.00th=[55552], 99.50th=[110080], 99.90th=[199680], 99.95th=[199680],
     | 99.99th=[220160]
    bw (KB  /s): min=    1, max=421376, per=100.00%, avg=122938.45, stdev=50653.61
    lat (usec) : 100=0.01%, 250=0.28%, 500=56.05%, 750=36.75%, 1000=3.13%
    lat (msec) : 2=1.50%, 4=0.31%, 10=0.13%, 20=0.06%, 50=0.67%
    lat (msec) : 100=0.55%, 250=0.57%, 500=0.01%, 750=0.01%
  cpu          : usr=1.10%, sys=3.34%, ctx=22693155, majf=0, minf=9437
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13725458/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=204336: Thu Jan 14 11:14:28 2016
  write: io=845895MB, bw=120304KB/s, iops=1879, runt=7200055msec
    slat (usec): min=7, max=329, avg=25.56, stdev=12.15
    clat (usec): min=134, max=403655, avg=2100.92, stdev=13751.11
     lat (usec): min=207, max=403677, avg=2126.65, stdev=13751.07
    clat percentiles (usec):
     |  1.00th=[  294],  5.00th=[  334], 10.00th=[  354], 20.00th=[  398],
     | 30.00th=[  438], 40.00th=[  462], 50.00th=[  478], 60.00th=[  498],
     | 70.00th=[  540], 80.00th=[  596], 90.00th=[  692], 95.00th=[  884],
     | 99.00th=[57600], 99.50th=[112128], 99.90th=[199680], 99.95th=[199680],
     | 99.99th=[226304]
    bw (KB  /s): min=    1, max=451456, per=100.00%, avg=121281.21, stdev=50946.48
    lat (usec) : 250=0.16%, 500=59.99%, 750=32.39%, 1000=3.48%
    lat (msec) : 2=1.64%, 4=0.31%, 10=0.13%, 20=0.05%, 50=0.66%
    lat (msec) : 100=0.59%, 250=0.59%, 500=0.01%
  cpu          : usr=0.88%, sys=2.40%, ctx=23102275, majf=0, minf=9716
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13534324/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=2370.8GB, aggrb=345263KB/s, minb=345263KB/s, maxb=345263KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=2792.3GB, aggrb=406655KB/s, minb=406655KB/s, maxb=406655KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=857841MB, aggrb=122004KB/s, minb=122004KB/s, maxb=122004KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=845895MB, aggrb=120304KB/s, minb=120304KB/s, maxb=120304KB/s, mint=7200055msec, maxt=7200055msec



# 
# Thu 14 Jan 2016 11:14:31 AM EST
# 64kB read with SATA 1 jobs and 10 I/Os Outstanding
# 64kB read with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 7259 IOPS | 464 MBps | 1376 μsec Average Latency 
SATA-R5-1: 7279 IOPS | 465 MBps | 1372 μsec Average Latency 
NVMe-R5-0: 9307 IOPS | 595 MBps | 858 μsec Average Latency 
NVMe-R5-1: 9180 IOPS | 587 MBps | 870 μsec Average Latency 

Totals
IOPS: 33025
MBPS: 2111
Average Latency (μsec): 1119

SATA-R5-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
SATA-R5-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
NVMe-R5-01: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=209486: Thu Jan 14 13:15:31 2016
  read : io=3190.2GB, bw=464597KB/s, iops=7259, runt=7200001msec
    slat (usec): min=6, max=2293, avg=23.85, stdev=22.42
    clat (usec): min=1, max=23805, avg=1352.44, stdev=481.02
     lat (usec): min=187, max=23827, avg=1376.43, stdev=479.93
    clat percentiles (usec):
     |  1.00th=[  556],  5.00th=[  788], 10.00th=[  908], 20.00th=[ 1032],
     | 30.00th=[ 1128], 40.00th=[ 1224], 50.00th=[ 1288], 60.00th=[ 1384],
     | 70.00th=[ 1480], 80.00th=[ 1608], 90.00th=[ 1816], 95.00th=[ 2024],
     | 99.00th=[ 2992], 99.50th=[ 3600], 99.90th=[ 5856], 99.95th=[ 7392],
     | 99.99th=[10432]
    bw (KB  /s): min=    1, max=512512, per=100.00%, avg=464730.32, stdev=13871.41
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.62%, 750=3.32%, 1000=12.51%
    lat (msec) : 2=78.27%, 4=4.95%, 10=0.31%, 20=0.01%, 50=0.01%
  cpu          : usr=1.33%, sys=9.42%, ctx=77625529, majf=0, minf=22475
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=52267174/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=209487: Thu Jan 14 13:15:31 2016
  read : io=3199.1GB, bw=465901KB/s, iops=7279, runt=7200002msec
    slat (usec): min=5, max=2087, avg=18.19, stdev=17.59
    clat (usec): min=1, max=20872, avg=1354.32, stdev=530.12
     lat (usec): min=181, max=20883, avg=1372.65, stdev=529.49
    clat percentiles (usec):
     |  1.00th=[  524],  5.00th=[  756], 10.00th=[  884], 20.00th=[ 1020],
     | 30.00th=[ 1112], 40.00th=[ 1208], 50.00th=[ 1272], 60.00th=[ 1368],
     | 70.00th=[ 1480], 80.00th=[ 1608], 90.00th=[ 1832], 95.00th=[ 2096],
     | 99.00th=[ 3376], 99.50th=[ 3920], 99.90th=[ 6112], 99.95th=[ 7776],
     | 99.99th=[10688]
    bw (KB  /s): min=    1, max=517888, per=100.00%, avg=466035.19, stdev=15028.86
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.80%, 750=3.88%, 1000=13.37%
    lat (msec) : 2=75.78%, 4=5.70%, 10=0.44%, 20=0.02%, 50=0.01%
  cpu          : usr=1.34%, sys=7.92%, ctx=79354692, majf=0, minf=22127
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=52413881/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=209488: Thu Jan 14 13:15:31 2016
  read : io=4090.2GB, bw=595664KB/s, iops=9307, runt=7200001msec
    slat (usec): min=6, max=1474, avg=23.07, stdev=21.87
    clat (usec): min=0, max=19562, avg=835.26, stdev=360.07
     lat (usec): min=177, max=19596, avg=858.48, stdev=358.53
    clat percentiles (usec):
     |  1.00th=[  354],  5.00th=[  458], 10.00th=[  506], 20.00th=[  588],
     | 30.00th=[  652], 40.00th=[  716], 50.00th=[  772], 60.00th=[  844],
     | 70.00th=[  932], 80.00th=[ 1048], 90.00th=[ 1224], 95.00th=[ 1384],
     | 99.00th=[ 1816], 99.50th=[ 2064], 99.90th=[ 4048], 99.95th=[ 5920],
     | 99.99th=[ 9536]
    bw (KB  /s): min=    1, max=645632, per=100.00%, avg=595784.47, stdev=18456.54
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.10%, 500=9.17%, 750=37.03%, 1000=30.08%
    lat (msec) : 2=23.04%, 4=0.48%, 10=0.09%, 20=0.01%
  cpu          : usr=1.62%, sys=11.42%, ctx=96560894, majf=0, minf=17866
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=67012195/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=209489: Thu Jan 14 13:15:31 2016
  read : io=4034.7GB, bw=587582KB/s, iops=9180, runt=7200001msec
    slat (usec): min=5, max=1705, avg=23.93, stdev=22.99
    clat (usec): min=0, max=20482, avg=846.20, stdev=375.80
     lat (usec): min=172, max=20498, avg=870.29, stdev=374.23
    clat percentiles (usec):
     |  1.00th=[  346],  5.00th=[  450], 10.00th=[  498], 20.00th=[  580],
     | 30.00th=[  644], 40.00th=[  716], 50.00th=[  780], 60.00th=[  860],
     | 70.00th=[  956], 80.00th=[ 1080], 90.00th=[ 1256], 95.00th=[ 1432],
     | 99.00th=[ 1848], 99.50th=[ 2096], 99.90th=[ 4192], 99.95th=[ 6048],
     | 99.99th=[ 9920]
    bw (KB  /s): min=    1, max=643968, per=100.00%, avg=587697.89, stdev=19724.66
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.12%, 500=9.94%, 750=35.76%, 1000=28.49%
    lat (msec) : 2=25.04%, 4=0.54%, 10=0.10%, 20=0.01%, 50=0.01%
  cpu          : usr=1.62%, sys=11.23%, ctx=95970780, majf=0, minf=18552
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=66103026/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3190.2GB, aggrb=464597KB/s, minb=464597KB/s, maxb=464597KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=3199.1GB, aggrb=465901KB/s, minb=465901KB/s, maxb=465901KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=4090.2GB, aggrb=595663KB/s, minb=595663KB/s, maxb=595663KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=4034.7GB, aggrb=587582KB/s, minb=587582KB/s, maxb=587582KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 01:15:34 PM EST
# 64kB randread with SATA 2 jobs and 2 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 3680 IOPS | 235 MBps | 1084 μsec Average Latency 
SATA-R5-1: 3707 IOPS | 237 MBps | 1076 μsec Average Latency 
NVMe-R5-0: 8640 IOPS | 553 MBps | 923 μsec Average Latency 
NVMe-R5-1: 7353 IOPS | 470 MBps | 1085 μsec Average Latency 

Totals
IOPS: 23380
MBPS: 1495
Average Latency (μsec): 1042

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 12 processes

SATA-R5-01: (groupid=0, jobs=2): err= 0: pid=214607: Thu Jan 14 15:16:35 2016
  read : io=1617.6GB, bw=235569KB/s, iops=3680, runt=7200001msec
    slat (usec): min=6, max=913, avg=33.62, stdev=29.89
    clat (usec): min=61, max=17387, avg=1050.22, stdev=340.39
     lat (usec): min=204, max=17422, avg=1084.05, stdev=338.23
    clat percentiles (usec):
     |  1.00th=[  676],  5.00th=[  764], 10.00th=[  812], 20.00th=[  876],
     | 30.00th=[  924], 40.00th=[  972], 50.00th=[ 1012], 60.00th=[ 1064],
     | 70.00th=[ 1112], 80.00th=[ 1176], 90.00th=[ 1272], 95.00th=[ 1384],
     | 99.00th=[ 1864], 99.50th=[ 2992], 99.90th=[ 5600], 99.95th=[ 6560],
     | 99.99th=[10048]
    bw (KB  /s): min=    1, max=124672, per=50.03%, avg=117858.39, stdev=2827.47
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.04%, 750=4.09%, 1000=43.18%
    lat (msec) : 2=51.79%, 4=0.70%, 10=0.20%, 20=0.01%
  cpu          : usr=0.79%, sys=3.62%, ctx=50995234, majf=0, minf=11863
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26501473/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=2): err= 0: pid=214609: Thu Jan 14 15:16:35 2016
  read : io=1629.7GB, bw=237249KB/s, iops=3707, runt=7200001msec
    slat (usec): min=6, max=1210, avg=32.28, stdev=27.43
    clat (usec): min=1, max=21129, avg=1043.86, stdev=335.87
     lat (usec): min=200, max=21143, avg=1076.36, stdev=334.14
    clat percentiles (usec):
     |  1.00th=[  620],  5.00th=[  732], 10.00th=[  796], 20.00th=[  868],
     | 30.00th=[  924], 40.00th=[  972], 50.00th=[ 1012], 60.00th=[ 1064],
     | 70.00th=[ 1112], 80.00th=[ 1176], 90.00th=[ 1272], 95.00th=[ 1384],
     | 99.00th=[ 1720], 99.50th=[ 2640], 99.90th=[ 5600], 99.95th=[ 6624],
     | 99.99th=[10560]
    bw (KB  /s): min=    1, max=129664, per=50.03%, avg=118699.25, stdev=3285.57
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.08%
    lat (usec) : 750=6.27%, 1000=40.52%
    lat (msec) : 2=52.44%, 4=0.50%, 10=0.18%, 20=0.01%, 50=0.01%
  cpu          : usr=0.80%, sys=3.62%, ctx=51473676, majf=0, minf=11604
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26690535/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=214611: Thu Jan 14 15:16:35 2016
  read : io=3797.3GB, bw=553015KB/s, iops=8640, runt=7200001msec
    slat (usec): min=6, max=1633, avg=32.96, stdev=28.91
    clat (usec): min=0, max=20842, avg=890.17, stdev=358.08
     lat (usec): min=172, max=20857, avg=923.34, stdev=356.26
    clat percentiles (usec):
     |  1.00th=[  338],  5.00th=[  498], 10.00th=[  596], 20.00th=[  684],
     | 30.00th=[  740], 40.00th=[  796], 50.00th=[  844], 60.00th=[  900],
     | 70.00th=[  972], 80.00th=[ 1064], 90.00th=[ 1224], 95.00th=[ 1384],
     | 99.00th=[ 1832], 99.50th=[ 2128], 99.90th=[ 5280], 99.95th=[ 6432],
     | 99.99th=[ 9920]
    bw (KB  /s): min=    1, max=183936, per=25.01%, avg=138325.91, stdev=6561.05
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.18%, 500=4.93%, 750=26.47%, 1000=42.38%
    lat (msec) : 2=25.40%, 4=0.48%, 10=0.15%, 20=0.01%, 50=0.01%
  cpu          : usr=0.88%, sys=4.33%, ctx=119035701, majf=0, minf=24051
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=62214243/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=214615: Thu Jan 14 15:16:35 2016
  read : io=3231.6GB, bw=470629KB/s, iops=7353, runt=7200001msec
    slat (usec): min=6, max=1709, avg=33.66, stdev=28.38
    clat (usec): min=0, max=20893, avg=1051.37, stdev=658.77
     lat (usec): min=173, max=20930, avg=1085.26, stdev=657.89
    clat percentiles (usec):
     |  1.00th=[  326],  5.00th=[  494], 10.00th=[  588], 20.00th=[  676],
     | 30.00th=[  740], 40.00th=[  796], 50.00th=[  860], 60.00th=[  940],
     | 70.00th=[ 1048], 80.00th=[ 1240], 90.00th=[ 1768], 95.00th=[ 2320],
     | 99.00th=[ 3696], 99.50th=[ 4384], 99.90th=[ 6112], 99.95th=[ 7008],
     | 99.99th=[10944]
    bw (KB  /s): min=    1, max=157568, per=25.02%, avg=117728.25, stdev=13053.36
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.22%, 500=5.03%, 750=26.17%, 1000=34.86%
    lat (msec) : 2=26.18%, 4=6.82%, 10=0.71%, 20=0.02%, 50=0.01%
  cpu          : usr=0.79%, sys=3.76%, ctx=102202244, majf=0, minf=23628
  IO depths    : 1=0.1%, 2=101.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=52945820/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1617.6GB, aggrb=235568KB/s, minb=235568KB/s, maxb=235568KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1629.7GB, aggrb=237249KB/s, minb=237249KB/s, maxb=237249KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=3797.3GB, aggrb=553015KB/s, minb=553015KB/s, maxb=553015KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=3231.6GB, aggrb=470629KB/s, minb=470629KB/s, maxb=470629KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 03:16:37 PM EST
# 64kB randread with SATA 6 jobs and 4 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6556 IOPS | 419 MBps | 3658 μsec Average Latency 
SATA-R5-1: 7669 IOPS | 490 MBps | 3126 μsec Average Latency 
NVMe-R5-0: 9473 IOPS | 606 MBps | 3375 μsec Average Latency 
NVMe-R5-1: 7897 IOPS | 505 MBps | 4049 μsec Average Latency 

Totals
IOPS: 31595
MBPS: 2020
Average Latency (μsec): 3552

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=219794: Thu Jan 14 17:17:38 2016
  read : io=2881.3GB, bw=419612KB/s, iops=6556, runt=7200003msec
    slat (usec): min=5, max=4304, avg=27.35, stdev=36.30
    clat (usec): min=0, max=24675, avg=3630.52, stdev=1592.22
     lat (usec): min=247, max=24726, avg=3658.08, stdev=1591.50
    clat percentiles (usec):
     |  1.00th=[ 1416],  5.00th=[ 2224], 10.00th=[ 2544], 20.00th=[ 2768],
     | 30.00th=[ 2896], 40.00th=[ 2992], 50.00th=[ 3120], 60.00th=[ 3248],
     | 70.00th=[ 3472], 80.00th=[ 4080], 90.00th=[ 5856], 95.00th=[ 7264],
     | 99.00th=[ 9664], 99.50th=[10688], 99.90th=[12864], 99.95th=[13760],
     | 99.99th=[15808]
    bw (KB  /s): min=    1, max=80512, per=16.68%, avg=69987.78, stdev=1855.79
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.14%
    lat (msec) : 2=3.23%, 4=75.85%, 10=19.98%, 20=0.79%, 50=0.01%
  cpu          : usr=0.44%, sys=1.99%, ctx=91692039, majf=0, minf=64993
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47206351/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=219800: Thu Jan 14 17:17:38 2016
  read : io=3370.4GB, bw=490836KB/s, iops=7669, runt=7200003msec
    slat (usec): min=6, max=3215, avg=34.53, stdev=39.02
    clat (usec): min=0, max=24932, avg=3091.86, stdev=1618.39
     lat (usec): min=248, max=24977, avg=3126.65, stdev=1617.75
    clat percentiles (usec):
     |  1.00th=[ 1256],  5.00th=[ 1640], 10.00th=[ 1864], 20.00th=[ 2096],
     | 30.00th=[ 2288], 40.00th=[ 2448], 50.00th=[ 2608], 60.00th=[ 2800],
     | 70.00th=[ 3056], 80.00th=[ 3568], 90.00th=[ 5280], 95.00th=[ 6816],
     | 99.00th=[ 9280], 99.50th=[10304], 99.90th=[12480], 99.95th=[13376],
     | 99.99th=[15424]
    bw (KB  /s): min=    1, max=104192, per=16.68%, avg=81865.46, stdev=5178.53
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.20%
    lat (msec) : 2=14.91%, 4=69.11%, 10=15.17%, 20=0.60%, 50=0.01%
  cpu          : usr=0.57%, sys=2.52%, ctx=105073547, majf=0, minf=64732
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=55219034/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=219806: Thu Jan 14 17:17:38 2016
  read : io=4163.3GB, bw=606284KB/s, iops=9473, runt=7200003msec
    slat (usec): min=5, max=4828, avg=27.56, stdev=35.90
    clat (usec): min=1, max=29566, avg=3348.11, stdev=1579.67
     lat (usec): min=217, max=29583, avg=3375.87, stdev=1578.95
    clat percentiles (usec):
     |  1.00th=[  996],  5.00th=[ 1752], 10.00th=[ 2160], 20.00th=[ 2480],
     | 30.00th=[ 2640], 40.00th=[ 2768], 50.00th=[ 2864], 60.00th=[ 3024],
     | 70.00th=[ 3280], 80.00th=[ 3888], 90.00th=[ 5472], 95.00th=[ 6880],
     | 99.00th=[ 9280], 99.50th=[10304], 99.90th=[12608], 99.95th=[13504],
     | 99.99th=[15680]
    bw (KB  /s): min=    1, max=173440, per=25.01%, avg=151658.62, stdev=5640.16
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.06%, 750=0.26%, 1000=0.70%
    lat (msec) : 2=6.63%, 4=73.42%, 10=18.33%, 20=0.61%, 50=0.01%
  cpu          : usr=0.79%, sys=4.08%, ctx=125018281, majf=0, minf=80588
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=68206917/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=219810: Thu Jan 14 17:17:38 2016
  read : io=3470.7GB, bw=505447KB/s, iops=7897, runt=7200004msec
    slat (usec): min=5, max=3714, avg=31.85, stdev=39.38
    clat (usec): min=0, max=75143, avg=4017.38, stdev=3443.24
     lat (usec): min=189, max=75156, avg=4049.49, stdev=3443.00
    clat percentiles (usec):
     |  1.00th=[  732],  5.00th=[ 1128], 10.00th=[ 1416], 20.00th=[ 1736],
     | 30.00th=[ 1960], 40.00th=[ 2192], 50.00th=[ 2448], 60.00th=[ 2960],
     | 70.00th=[ 4576], 80.00th=[ 6560], 90.00th=[ 8512], 95.00th=[10560],
     | 99.00th=[17024], 99.50th=[19328], 99.90th=[24704], 99.95th=[27264],
     | 99.99th=[34048]
    bw (KB  /s): min=    1, max=171984, per=25.02%, avg=126442.22, stdev=11943.60
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.23%, 750=0.90%, 1000=2.18%
    lat (msec) : 2=28.36%, 4=35.66%, 10=26.69%, 20=5.57%, 50=0.40%
    lat (msec) : 100=0.01%
  cpu          : usr=0.75%, sys=3.59%, ctx=104762971, majf=0, minf=76428
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=101.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=56862754/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=2881.3GB, aggrb=419611KB/s, minb=419611KB/s, maxb=419611KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=3370.4GB, aggrb=490835KB/s, minb=490835KB/s, maxb=490835KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=4163.3GB, aggrb=606283KB/s, minb=606283KB/s, maxb=606283KB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=3470.7GB, aggrb=505446KB/s, minb=505446KB/s, maxb=505446KB/s, mint=7200004msec, maxt=7200004msec



# 
# Thu 14 Jan 2016 05:17:40 PM EST
# 32kB randread with SATA 5 jobs and 2 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 9515 IOPS | 304 MBps | 1048 μsec Average Latency 
SATA-R5-1: 9926 IOPS | 317 MBps | 1005 μsec Average Latency 
NVMe-R5-0: 13670 IOPS | 437 MBps | 876 μsec Average Latency 
NVMe-R5-1: 12967 IOPS | 414 MBps | 923 μsec Average Latency 

Totals
IOPS: 46078
MBPS: 1472
Average Latency (μsec): 963

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=224930: Thu Jan 14 19:18:41 2016
  read : io=2090.9GB, bw=304497KB/s, iops=9515, runt=7200001msec
    slat (usec): min=4, max=2358, avg=26.01, stdev=27.87
    clat (usec): min=1, max=23835, avg=1022.37, stdev=539.32
     lat (usec): min=172, max=23851, avg=1048.59, stdev=538.06
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  692], 10.00th=[  748], 20.00th=[  812],
     | 30.00th=[  860], 40.00th=[  900], 50.00th=[  948], 60.00th=[  996],
     | 70.00th=[ 1048], 80.00th=[ 1128], 90.00th=[ 1256], 95.00th=[ 1400],
     | 99.00th=[ 3184], 99.50th=[ 4896], 99.90th=[ 8256], 99.95th=[ 9920],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=67136, per=20.01%, avg=60937.42, stdev=2389.11
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.20%, 750=10.52%, 1000=50.20%
    lat (msec) : 2=37.56%, 4=0.71%, 10=0.76%, 20=0.05%, 50=0.01%
  cpu          : usr=0.72%, sys=2.91%, ctx=130922589, majf=0, minf=18273
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=68511815/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=224935: Thu Jan 14 19:18:41 2016
  read : io=2181.8GB, bw=317632KB/s, iops=9926, runt=7200007msec
    slat (usec): min=3, max=1880, avg=18.91, stdev=22.84
    clat (usec): min=1, max=22500, avg=986.49, stdev=539.92
     lat (usec): min=182, max=22524, avg=1005.56, stdev=539.29
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  652], 10.00th=[  708], 20.00th=[  772],
     | 30.00th=[  820], 40.00th=[  868], 50.00th=[  908], 60.00th=[  964],
     | 70.00th=[ 1012], 80.00th=[ 1096], 90.00th=[ 1224], 95.00th=[ 1368],
     | 99.00th=[ 3088], 99.50th=[ 4832], 99.90th=[ 8256], 99.95th=[10048],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=71552, per=20.01%, avg=63564.90, stdev=2326.06
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.29%, 750=16.32%, 1000=50.90%
    lat (msec) : 2=31.02%, 4=0.69%, 10=0.73%, 20=0.05%, 50=0.01%
  cpu          : usr=0.62%, sys=2.45%, ctx=137983381, majf=0, minf=18300
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=71467276/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=224940: Thu Jan 14 19:18:41 2016
  read : io=3003.8GB, bw=437446KB/s, iops=13670, runt=7200001msec
    slat (usec): min=4, max=1742, avg=23.72, stdev=26.14
    clat (usec): min=1, max=24381, avg=852.12, stdev=521.20
     lat (usec): min=147, max=24392, avg=876.03, stdev=519.74
    clat percentiles (usec):
     |  1.00th=[  382],  5.00th=[  502], 10.00th=[  564], 20.00th=[  636],
     | 30.00th=[  692], 40.00th=[  732], 50.00th=[  772], 60.00th=[  828],
     | 70.00th=[  884], 80.00th=[  972], 90.00th=[ 1112], 95.00th=[ 1288],
     | 99.00th=[ 2352], 99.50th=[ 4576], 99.90th=[ 7840], 99.95th=[ 9536],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=172864, per=33.34%, avg=145863.24, stdev=7335.32
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.08%, 500=4.89%, 750=38.95%, 1000=38.92%
    lat (msec) : 2=15.96%, 4=0.53%, 10=0.63%, 20=0.04%, 50=0.01%
  cpu          : usr=1.28%, sys=5.90%, ctx=169525964, majf=0, minf=17688
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=98425459/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=224943: Thu Jan 14 19:18:41 2016
  read : io=2849.4GB, bw=414971KB/s, iops=12967, runt=7200001msec
    slat (usec): min=3, max=2998, avg=19.47, stdev=22.84
    clat (usec): min=0, max=24610, avg=903.99, stdev=613.54
     lat (usec): min=136, max=24619, avg=923.64, stdev=612.88
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  502], 10.00th=[  572], 20.00th=[  636],
     | 30.00th=[  692], 40.00th=[  740], 50.00th=[  780], 60.00th=[  836],
     | 70.00th=[  908], 80.00th=[ 1012], 90.00th=[ 1240], 95.00th=[ 1544],
     | 99.00th=[ 3760], 99.50th=[ 4896], 99.90th=[ 8096], 99.95th=[ 9920],
     | 99.99th=[12352]
    bw (KB  /s): min=    0, max=183296, per=33.35%, avg=138373.60, stdev=12019.59
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.08%, 500=4.90%, 750=38.55%, 1000=35.40%
    lat (msec) : 2=18.29%, 4=1.90%, 10=0.83%, 20=0.05%, 50=0.01%
  cpu          : usr=1.14%, sys=4.87%, ctx=164750519, majf=0, minf=19303
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=93368380/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=2090.9GB, aggrb=304496KB/s, minb=304496KB/s, maxb=304496KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=2181.8GB, aggrb=317632KB/s, minb=317632KB/s, maxb=317632KB/s, mint=7200007msec, maxt=7200007msec

Run status group 2 (all jobs):
   READ: io=3003.8GB, aggrb=437446KB/s, minb=437446KB/s, maxb=437446KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=2849.4GB, aggrb=414970KB/s, minb=414970KB/s, maxb=414970KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 07:18:43 PM EST
# 32kB randread with SATA 4 jobs and 16 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 17777 IOPS | 568 MBps | 3598 μsec Average Latency 
SATA-R5-1: 17747 IOPS | 567 MBps | 3604 μsec Average Latency 
NVMe-R5-0: 9656 IOPS | 308 MBps | 2483 μsec Average Latency 
NVMe-R5-1: 9483 IOPS | 303 MBps | 2528 μsec Average Latency 

Totals
IOPS: 54663
MBPS: 1746
Average Latency (μsec): 3053

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 14 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=230103: Thu Jan 14 21:19:44 2016
  read : io=3906.3GB, bw=568894KB/s, iops=17777, runt=7200007msec
    slat (usec): min=4, max=5941, avg=23.24, stdev=39.72
    clat (usec): min=0, max=108375, avg=3574.87, stdev=1582.86
     lat (usec): min=183, max=108386, avg=3598.28, stdev=1582.07
    clat percentiles (usec):
     |  1.00th=[ 1528],  5.00th=[ 2024], 10.00th=[ 2224], 20.00th=[ 2512],
     | 30.00th=[ 2736], 40.00th=[ 2960], 50.00th=[ 3184], 60.00th=[ 3408],
     | 70.00th=[ 3728], 80.00th=[ 4192], 90.00th=[ 5408], 95.00th=[ 7136],
     | 99.00th=[ 9536], 99.50th=[10560], 99.90th=[12864], 99.95th=[13760],
     | 99.99th=[15808]
    bw (KB  /s): min=    0, max=165696, per=25.01%, avg=142285.34, stdev=4836.38
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.07%
    lat (msec) : 2=4.56%, 4=71.80%, 10=22.84%, 20=0.72%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.09%, sys=5.39%, ctx=211616468, majf=0, minf=81010
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=128001125/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=230107: Thu Jan 14 21:19:44 2016
  read : io=3899.7GB, bw=567926KB/s, iops=17747, runt=7200004msec
    slat (usec): min=3, max=4099, avg=23.97, stdev=36.28
    clat (usec): min=0, max=118254, avg=3580.31, stdev=1587.34
     lat (usec): min=266, max=118279, avg=3604.46, stdev=1586.66
    clat percentiles (usec):
     |  1.00th=[ 1528],  5.00th=[ 2008], 10.00th=[ 2224], 20.00th=[ 2512],
     | 30.00th=[ 2736], 40.00th=[ 2960], 50.00th=[ 3184], 60.00th=[ 3440],
     | 70.00th=[ 3728], 80.00th=[ 4192], 90.00th=[ 5472], 95.00th=[ 7136],
     | 99.00th=[ 9536], 99.50th=[10560], 99.90th=[12992], 99.95th=[13888],
     | 99.99th=[15936]
    bw (KB  /s): min=    0, max=167680, per=25.01%, avg=142043.44, stdev=4678.09
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.07%
    lat (msec) : 2=4.68%, 4=71.55%, 10=22.96%, 20=0.73%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.11%, sys=5.36%, ctx=208971979, majf=0, minf=83139
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=127783280/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=230111: Thu Jan 14 21:19:44 2016
  read : io=2121.8GB, bw=308997KB/s, iops=9656, runt=7200005msec
    slat (usec): min=4, max=6395, avg=21.20, stdev=39.71
    clat (usec): min=0, max=100764, avg=2462.34, stdev=1428.39
     lat (usec): min=160, max=100782, avg=2483.71, stdev=1427.86
    clat percentiles (usec):
     |  1.00th=[  820],  5.00th=[ 1224], 10.00th=[ 1416], 20.00th=[ 1608],
     | 30.00th=[ 1752], 40.00th=[ 1896], 50.00th=[ 2064], 60.00th=[ 2256],
     | 70.00th=[ 2512], 80.00th=[ 2960], 90.00th=[ 3760], 95.00th=[ 5728],
     | 99.00th=[ 8256], 99.50th=[ 9280], 99.90th=[11584], 99.95th=[12480],
     | 99.99th=[14400]
    bw (KB  /s): min=    0, max=121024, per=33.35%, avg=103053.03, stdev=4318.05
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.05%, 750=0.58%, 1000=1.63%
    lat (msec) : 2=44.76%, 4=44.57%, 10=8.09%, 20=0.31%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.86%, sys=3.73%, ctx=121064926, majf=0, minf=32601
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=69524337/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=230114: Thu Jan 14 21:19:44 2016
  read : io=2083.9GB, bw=303474KB/s, iops=9483, runt=7200002msec
    slat (usec): min=3, max=4985, avg=23.84, stdev=36.82
    clat (usec): min=0, max=117676, avg=2504.86, stdev=1439.69
     lat (usec): min=197, max=117689, avg=2528.90, stdev=1439.21
    clat percentiles (usec):
     |  1.00th=[  844],  5.00th=[ 1240], 10.00th=[ 1432], 20.00th=[ 1640],
     | 30.00th=[ 1784], 40.00th=[ 1928], 50.00th=[ 2096], 60.00th=[ 2288],
     | 70.00th=[ 2576], 80.00th=[ 3024], 90.00th=[ 3792], 95.00th=[ 5856],
     | 99.00th=[ 8384], 99.50th=[ 9280], 99.90th=[11584], 99.95th=[12480],
     | 99.99th=[14528]
    bw (KB  /s): min=    0, max=119424, per=33.35%, avg=101210.55, stdev=4296.82
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.04%, 750=0.49%, 1000=1.56%
    lat (msec) : 2=42.23%, 4=46.98%, 10=8.38%, 20=0.32%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.90%, sys=3.82%, ctx=117706515, majf=0, minf=33165
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=68281738/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3906.3GB, aggrb=568893KB/s, minb=568893KB/s, maxb=568893KB/s, mint=7200007msec, maxt=7200007msec

Run status group 1 (all jobs):
   READ: io=3899.7GB, aggrb=567925KB/s, minb=567925KB/s, maxb=567925KB/s, mint=7200004msec, maxt=7200004msec

Run status group 2 (all jobs):
   READ: io=2121.8GB, aggrb=308996KB/s, minb=308996KB/s, maxb=308996KB/s, mint=7200005msec, maxt=7200005msec

Run status group 3 (all jobs):
   READ: io=2083.9GB, aggrb=303474KB/s, minb=303474KB/s, maxb=303474KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:19:47 PM EST
# 16kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 16kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 28268 IOPS | 452 MBps | 2828 μsec Average Latency 
SATA-R5-1: 27978 IOPS | 447 MBps | 2857 μsec Average Latency 
NVMe-R5-0: 6491 IOPS | 103 MBps | 2462 μsec Average Latency 
NVMe-R5-1: 6650 IOPS | 106 MBps | 2404 μsec Average Latency 

Totals
IOPS: 69387
MBPS: 1108
Average Latency (μsec): 2637

SATA-R5-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=235243: Thu Jan 14 23:20:48 2016
  read : io=3105.8GB, bw=452302KB/s, iops=28268, runt=7200005msec
    slat (usec): min=3, max=2472, avg=20.57, stdev=23.50
    clat (usec): min=66, max=164135, avg=2807.53, stdev=599.33
     lat (usec): min=382, max=164219, avg=2828.29, stdev=598.03
    clat percentiles (usec):
     |  1.00th=[ 1848],  5.00th=[ 2128], 10.00th=[ 2256], 20.00th=[ 2416],
     | 30.00th=[ 2544], 40.00th=[ 2640], 50.00th=[ 2736], 60.00th=[ 2864],
     | 70.00th=[ 2960], 80.00th=[ 3120], 90.00th=[ 3344], 95.00th=[ 3600],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8256], 99.95th=[ 9152],
     | 99.99th=[12096]
    bw (KB  /s): min=    0, max=96224, per=20.01%, avg=90487.37, stdev=1850.62
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=2.34%, 4=95.48%, 10=2.14%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.44%, sys=6.32%, ctx=336636784, majf=0, minf=51901
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=203536021/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=235248: Thu Jan 14 23:20:48 2016
  read : io=3073.9GB, bw=447658KB/s, iops=27978, runt=7200006msec
    slat (usec): min=2, max=3763, avg=14.26, stdev=18.21
    clat (usec): min=125, max=144923, avg=2843.46, stdev=592.58
     lat (usec): min=295, max=144931, avg=2857.85, stdev=591.99
    clat percentiles (usec):
     |  1.00th=[ 1880],  5.00th=[ 2160], 10.00th=[ 2320], 20.00th=[ 2480],
     | 30.00th=[ 2576], 40.00th=[ 2672], 50.00th=[ 2768], 60.00th=[ 2896],
     | 70.00th=[ 2992], 80.00th=[ 3152], 90.00th=[ 3376], 95.00th=[ 3632],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8256], 99.95th=[ 9152],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=95392, per=20.01%, avg=89558.86, stdev=1772.40
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=1.90%, 4=95.88%, 10=2.18%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.18%, sys=4.65%, ctx=340595537, majf=0, minf=52532
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=201445985/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=235253: Thu Jan 14 23:20:48 2016
  read : io=730252MB, bw=103858KB/s, iops=6491, runt=7200002msec
    slat (usec): min=2, max=2449, avg=22.48, stdev=25.50
    clat (usec): min=1, max=158308, avg=2439.73, stdev=581.69
     lat (usec): min=286, max=158319, avg=2462.43, stdev=580.65
    clat percentiles (usec):
     |  1.00th=[ 1496],  5.00th=[ 1784], 10.00th=[ 1912], 20.00th=[ 2064],
     | 30.00th=[ 2192], 40.00th=[ 2288], 50.00th=[ 2384], 60.00th=[ 2480],
     | 70.00th=[ 2608], 80.00th=[ 2736], 90.00th=[ 2992], 95.00th=[ 3216],
     | 99.00th=[ 4192], 99.50th=[ 5088], 99.90th=[ 7776], 99.95th=[ 8640],
     | 99.99th=[11456]
    bw (KB  /s): min=    0, max=28710, per=25.02%, avg=25981.83, stdev=670.63
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.01%, 750=0.02%, 1000=0.06%
    lat (msec) : 2=15.00%, 4=83.71%, 10=1.18%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.65%, sys=2.07%, ctx=89088410, majf=0, minf=16058
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=46736112/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=235257: Thu Jan 14 23:20:48 2016
  read : io=748159MB, bw=106405KB/s, iops=6650, runt=7200002msec
    slat (usec): min=2, max=2359, avg=13.53, stdev=17.55
    clat (usec): min=31, max=173324, avg=2390.39, stdev=576.37
     lat (usec): min=224, max=173335, avg=2404.05, stdev=575.97
    clat percentiles (usec):
     |  1.00th=[ 1448],  5.00th=[ 1720], 10.00th=[ 1864], 20.00th=[ 2008],
     | 30.00th=[ 2128], 40.00th=[ 2224], 50.00th=[ 2320], 60.00th=[ 2448],
     | 70.00th=[ 2544], 80.00th=[ 2704], 90.00th=[ 2928], 95.00th=[ 3152],
     | 99.00th=[ 4080], 99.50th=[ 5024], 99.90th=[ 7648], 99.95th=[ 8640],
     | 99.99th=[11072]
    bw (KB  /s): min=    0, max=29792, per=25.02%, avg=26618.51, stdev=775.47
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.03%
    lat (usec) : 1000=0.08%
    lat (msec) : 2=18.71%, 4=80.10%, 10=1.07%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.47%, sys=1.44%, ctx=87206799, majf=0, minf=14923
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47882140/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=3105.8GB, aggrb=452302KB/s, minb=452302KB/s, maxb=452302KB/s, mint=7200005msec, maxt=7200005msec

Run status group 1 (all jobs):
   READ: io=3073.9GB, aggrb=447657KB/s, minb=447657KB/s, maxb=447657KB/s, mint=7200006msec, maxt=7200006msec

Run status group 2 (all jobs):
   READ: io=730252MB, aggrb=103858KB/s, minb=103858KB/s, maxb=103858KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=748159MB, aggrb=106404KB/s, minb=106404KB/s, maxb=106404KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 11:20:50 PM EST
# 8kB randread with SATA 6 jobs and 2 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 19396 IOPS | 155 MBps | 616 μsec Average Latency 
SATA-R5-1: 18937 IOPS | 151 MBps | 631 μsec Average Latency 
NVMe-R5-0: 14300 IOPS | 114 MBps | 557 μsec Average Latency 
NVMe-R5-1: 13889 IOPS | 111 MBps | 573 μsec Average Latency 

Totals
IOPS: 66522
MBPS: 531
Average Latency (μsec): 594

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=240430: Fri Jan 15 01:21:51 2016
  read : io=1065.5GB, bw=155169KB/s, iops=19396, runt=7200001msec
    slat (usec): min=2, max=682, avg=11.29, stdev=10.97
    clat (usec): min=1, max=26387, avg=605.47, stdev=200.70
     lat (usec): min=160, max=26396, avg=616.87, stdev=200.23
    clat percentiles (usec):
     |  1.00th=[  402],  5.00th=[  458], 10.00th=[  478], 20.00th=[  494],
     | 30.00th=[  516], 40.00th=[  564], 50.00th=[  596], 60.00th=[  612],
     | 70.00th=[  628], 80.00th=[  660], 90.00th=[  732], 95.00th=[  812],
     | 99.00th=[ 1192], 99.50th=[ 1432], 99.90th=[ 3120], 99.95th=[ 3536],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=26944, per=16.67%, avg=25870.71, stdev=418.64
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=22.71%, 750=69.47%, 1000=5.48%
    lat (msec) : 2=2.02%, 4=0.29%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=0.83%, sys=2.39%, ctx=267159164, majf=0, minf=13220
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=139651782/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=240436: Fri Jan 15 01:21:51 2016
  read : io=1040.3GB, bw=151503KB/s, iops=18937, runt=7200002msec
    slat (usec): min=2, max=6902, avg=19.83, stdev=18.51
    clat (usec): min=17, max=74836, avg=611.44, stdev=195.94
     lat (usec): min=151, max=74843, avg=631.53, stdev=194.31
    clat percentiles (usec):
     |  1.00th=[  378],  5.00th=[  450], 10.00th=[  474], 20.00th=[  502],
     | 30.00th=[  532], 40.00th=[  572], 50.00th=[  596], 60.00th=[  612],
     | 70.00th=[  636], 80.00th=[  676], 90.00th=[  740], 95.00th=[  828],
     | 99.00th=[ 1192], 99.50th=[ 1368], 99.90th=[ 3056], 99.95th=[ 3536],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=27008, per=16.67%, avg=25259.74, stdev=426.14
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=19.45%
    lat (usec) : 750=71.61%, 1000=6.57%
    lat (msec) : 2=2.09%, 4=0.23%, 10=0.03%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.06%, sys=3.73%, ctx=259339674, majf=0, minf=16104
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=136352690/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=240442: Fri Jan 15 01:21:51 2016
  read : io=804429MB, bw=114408KB/s, iops=14300, runt=7200001msec
    slat (usec): min=2, max=15351, avg=11.04, stdev=10.64
    clat (usec): min=1, max=24881, avg=546.53, stdev=179.41
     lat (usec): min=106, max=24887, avg=557.68, stdev=178.81
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  378], 10.00th=[  430], 20.00th=[  466],
     | 30.00th=[  482], 40.00th=[  494], 50.00th=[  510], 60.00th=[  540],
     | 70.00th=[  588], 80.00th=[  620], 90.00th=[  668], 95.00th=[  748],
     | 99.00th=[ 1080], 99.50th=[ 1224], 99.90th=[ 2768], 99.95th=[ 3376],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=31664, per=25.01%, avg=28611.17, stdev=429.30
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.15%, 500=44.86%, 750=49.95%, 1000=3.57%
    lat (msec) : 2=1.29%, 4=0.15%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=0.88%, sys=2.52%, ctx=195786090, majf=0, minf=8764
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=102966958/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=240446: Fri Jan 15 01:21:51 2016
  read : io=781256MB, bw=111112KB/s, iops=13889, runt=7200001msec
    slat (usec): min=2, max=2180, avg=19.43, stdev=18.88
    clat (usec): min=8, max=22493, avg=554.21, stdev=181.19
     lat (usec): min=119, max=22520, avg=573.89, stdev=179.16
    clat percentiles (usec):
     |  1.00th=[  318],  5.00th=[  382], 10.00th=[  426], 20.00th=[  466],
     | 30.00th=[  482], 40.00th=[  502], 50.00th=[  524], 60.00th=[  556],
     | 70.00th=[  596], 80.00th=[  628], 90.00th=[  684], 95.00th=[  764],
     | 99.00th=[ 1096], 99.50th=[ 1240], 99.90th=[ 2768], 99.95th=[ 3376],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=29760, per=25.01%, avg=27787.14, stdev=448.75
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.21%
    lat (usec) : 500=38.86%, 750=55.40%, 1000=4.00%
    lat (msec) : 2=1.36%, 4=0.14%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=1.15%, sys=3.83%, ctx=189343933, majf=0, minf=11065
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=100000820/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1065.5GB, aggrb=155168KB/s, minb=155168KB/s, maxb=155168KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1040.3GB, aggrb=151502KB/s, minb=151502KB/s, maxb=151502KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=804429MB, aggrb=114407KB/s, minb=114407KB/s, maxb=114407KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=781256MB, aggrb=111112KB/s, minb=111112KB/s, maxb=111112KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 01:21:54 AM EST
# 8kB randread with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 24012 IOPS | 192 MBps | 1330 μsec Average Latency 
SATA-R5-1: 23900 IOPS | 191 MBps | 1337 μsec Average Latency 
NVMe-R5-0: 12873 IOPS | 102 MBps | 1240 μsec Average Latency 
NVMe-R5-1: 12919 IOPS | 103 MBps | 1236 μsec Average Latency 

Totals
IOPS: 73704
MBPS: 588
Average Latency (μsec): 1285

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=1092: Fri Jan 15 03:22:55 2016
  read : io=1319.4GB, bw=192098KB/s, iops=24012, runt=7200001msec
    slat (usec): min=2, max=7751, avg=17.81, stdev=18.59
    clat (usec): min=39, max=106251, avg=1312.93, stdev=417.42
     lat (usec): min=248, max=106260, avg=1330.94, stdev=416.22
    clat percentiles (usec):
     |  1.00th=[  756],  5.00th=[  916], 10.00th=[  988], 20.00th=[ 1080],
     | 30.00th=[ 1144], 40.00th=[ 1192], 50.00th=[ 1240], 60.00th=[ 1304],
     | 70.00th=[ 1368], 80.00th=[ 1464], 90.00th=[ 1672], 95.00th=[ 1880],
     | 99.00th=[ 2960], 99.50th=[ 3760], 99.90th=[ 5472], 99.95th=[ 6816],
     | 99.99th=[ 8384]
    bw (KB  /s): min=    0, max=53792, per=25.01%, avg=48034.67, stdev=911.08
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.03%, 750=0.93%
    lat (usec) : 1000=9.99%
    lat (msec) : 2=85.56%, 4=3.11%, 10=0.38%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.53%, sys=5.85%, ctx=295150564, majf=0, minf=15205
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172888311/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=1096: Fri Jan 15 03:22:55 2016
  read : io=1312.9GB, bw=191201KB/s, iops=23900, runt=7200001msec
    slat (usec): min=2, max=1586, avg=10.95, stdev=11.14
    clat (usec): min=52, max=82533, avg=1326.33, stdev=411.44
     lat (usec): min=227, max=82539, avg=1337.41, stdev=411.05
    clat percentiles (usec):
     |  1.00th=[  772],  5.00th=[  940], 10.00th=[ 1004], 20.00th=[ 1096],
     | 30.00th=[ 1144], 40.00th=[ 1224], 50.00th=[ 1256], 60.00th=[ 1320],
     | 70.00th=[ 1368], 80.00th=[ 1480], 90.00th=[ 1672], 95.00th=[ 1880],
     | 99.00th=[ 2896], 99.50th=[ 3728], 99.90th=[ 5408], 99.95th=[ 6816],
     | 99.99th=[ 8512]
    bw (KB  /s): min=    0, max=53904, per=25.01%, avg=47811.08, stdev=901.34
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.02%, 750=0.79%, 1000=8.65%
    lat (msec) : 2=87.05%, 4=3.11%, 10=0.37%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.26%, sys=3.93%, ctx=297479900, majf=0, minf=14941
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172080855/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=1100: Fri Jan 15 03:22:55 2016
  read : io=724121MB, bw=102986KB/s, iops=12873, runt=7200001msec
    slat (usec): min=2, max=2868, avg=18.59, stdev=19.15
    clat (usec): min=1, max=75257, avg=1222.02, stdev=382.32
     lat (usec): min=175, max=75271, avg=1240.83, stdev=381.18
    clat percentiles (usec):
     |  1.00th=[  684],  5.00th=[  844], 10.00th=[  916], 20.00th=[  996],
     | 30.00th=[ 1064], 40.00th=[ 1112], 50.00th=[ 1160], 60.00th=[ 1224],
     | 70.00th=[ 1288], 80.00th=[ 1368], 90.00th=[ 1560], 95.00th=[ 1752],
     | 99.00th=[ 2352], 99.50th=[ 3248], 99.90th=[ 5280], 99.95th=[ 6624],
     | 99.99th=[ 8096]
    bw (KB  /s): min=    0, max=28320, per=25.01%, avg=25756.36, stdev=541.84
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.09%, 750=2.08%, 1000=17.85%
    lat (msec) : 2=77.82%, 4=1.85%, 10=0.30%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.03%, sys=3.47%, ctx=172356979, majf=0, minf=10028
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=92687412/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=1104: Fri Jan 15 03:22:55 2016
  read : io=726709MB, bw=103354KB/s, iops=12919, runt=7200001msec
    slat (usec): min=2, max=1206, avg=11.23, stdev=11.93
    clat (usec): min=11, max=79257, avg=1225.39, stdev=377.82
     lat (usec): min=132, max=79263, avg=1236.76, stdev=377.43
    clat percentiles (usec):
     |  1.00th=[  692],  5.00th=[  852], 10.00th=[  932], 20.00th=[ 1004],
     | 30.00th=[ 1080], 40.00th=[ 1112], 50.00th=[ 1176], 60.00th=[ 1224],
     | 70.00th=[ 1272], 80.00th=[ 1368], 90.00th=[ 1560], 95.00th=[ 1752],
     | 99.00th=[ 2320], 99.50th=[ 3216], 99.90th=[ 5216], 99.95th=[ 6560],
     | 99.99th=[ 8256]
    bw (KB  /s): min=    0, max=27696, per=25.01%, avg=25848.29, stdev=504.93
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.09%
    lat (usec) : 750=1.95%, 1000=17.48%
    lat (msec) : 2=78.39%, 4=1.79%, 10=0.29%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.79%, sys=2.25%, ctx=174539780, majf=0, minf=10151
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=93018695/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=1319.4GB, aggrb=192098KB/s, minb=192098KB/s, maxb=192098KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1312.9GB, aggrb=191200KB/s, minb=191200KB/s, maxb=191200KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=724121MB, aggrb=102986KB/s, minb=102986KB/s, maxb=102986KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=726709MB, aggrb=103354KB/s, minb=103354KB/s, maxb=103354KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 03:22:58 AM EST
# 4kB randread with SATA 4 jobs and 2 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 15277 IOPS | 61 MBps | 521 μsec Average Latency 
SATA-R5-1: 14796 IOPS | 59 MBps | 538 μsec Average Latency 
NVMe-R5-0: 17054 IOPS | 68 MBps | 467 μsec Average Latency 
NVMe-R5-1: 16573 IOPS | 66 MBps | 480 μsec Average Latency 

Totals
IOPS: 63700
MBPS: 254
Average Latency (μsec): 501

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=6576: Fri Jan 15 05:23:59 2016
  read : io=429682MB, bw=61110KB/s, iops=15277, runt=7200001msec
    slat (usec): min=2, max=1160, avg= 9.72, stdev= 8.23
    clat (usec): min=19, max=61062, avg=512.05, stdev=161.09
     lat (usec): min=115, max=61068, avg=521.91, stdev=160.64
    clat percentiles (usec):
     |  1.00th=[  334],  5.00th=[  374], 10.00th=[  410], 20.00th=[  454],
     | 30.00th=[  474], 40.00th=[  482], 50.00th=[  490], 60.00th=[  502],
     | 70.00th=[  516], 80.00th=[  556], 90.00th=[  612], 95.00th=[  652],
     | 99.00th=[ 1020], 99.50th=[ 1224], 99.90th=[ 2480], 99.95th=[ 2960],
     | 99.99th=[ 6048]
    bw (KB  /s): min=    0, max=16808, per=25.01%, avg=15282.18, stdev=239.66
    lat (usec) : 20=0.01%, 100=0.01%, 250=0.04%, 500=59.86%, 750=37.66%
    lat (usec) : 1000=1.35%
    lat (msec) : 2=0.93%, 4=0.14%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.94%, sys=2.50%, ctx=206917644, majf=0, minf=6609
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=109998707/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=6580: Fri Jan 15 05:23:59 2016
  read : io=416159MB, bw=59187KB/s, iops=14796, runt=7200001msec
    slat (usec): min=2, max=2989, avg=17.89, stdev=16.03
    clat (usec): min=2, max=78238, avg=520.42, stdev=174.45
     lat (usec): min=100, max=78250, avg=538.53, stdev=172.99
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  374], 10.00th=[  410], 20.00th=[  450],
     | 30.00th=[  470], 40.00th=[  482], 50.00th=[  494], 60.00th=[  510],
     | 70.00th=[  540], 80.00th=[  580], 90.00th=[  620], 95.00th=[  676],
     | 99.00th=[ 1064], 99.50th=[ 1288], 99.90th=[ 2736], 99.95th=[ 3056],
     | 99.99th=[ 6304]
    bw (KB  /s): min=    0, max=15840, per=25.01%, avg=14801.33, stdev=277.38
    lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.12%, 500=52.80%
    lat (usec) : 750=44.31%, 1000=1.51%
    lat (msec) : 2=1.04%, 4=0.20%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.23%, sys=4.03%, ctx=199921458, majf=0, minf=6673
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=106536618/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=6584: Fri Jan 15 05:23:59 2016
  read : io=479662MB, bw=68219KB/s, iops=17054, runt=7200001msec
    slat (usec): min=2, max=15103, avg=10.13, stdev= 8.55
    clat (usec): min=1, max=66353, avg=457.25, stdev=141.99
     lat (usec): min=90, max=66359, avg=467.52, stdev=141.30
    clat percentiles (usec):
     |  1.00th=[  270],  5.00th=[  326], 10.00th=[  350], 20.00th=[  370],
     | 30.00th=[  390], 40.00th=[  430], 50.00th=[  466], 60.00th=[  482],
     | 70.00th=[  490], 80.00th=[  506], 90.00th=[  548], 95.00th=[  612],
     | 99.00th=[  892], 99.50th=[ 1048], 99.90th=[ 1448], 99.95th=[ 1928],
     | 99.99th=[ 5152]
    bw (KB  /s): min=    0, max=19056, per=25.01%, avg=17059.15, stdev=231.24
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.62%
    lat (usec) : 500=76.94%, 750=20.71%, 1000=1.09%
    lat (msec) : 2=0.59%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.97%, sys=2.78%, ctx=229998452, majf=0, minf=6439
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=122793460/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=6588: Fri Jan 15 05:23:59 2016
  read : io=466140MB, bw=66295KB/s, iops=16573, runt=7200001msec
    slat (usec): min=2, max=2492, avg=18.92, stdev=17.47
    clat (usec): min=0, max=57651, avg=461.51, stdev=143.03
     lat (usec): min=100, max=57670, avg=480.66, stdev=140.52
    clat percentiles (usec):
     |  1.00th=[  251],  5.00th=[  318], 10.00th=[  342], 20.00th=[  370],
     | 30.00th=[  402], 40.00th=[  438], 50.00th=[  462], 60.00th=[  478],
     | 70.00th=[  494], 80.00th=[  516], 90.00th=[  572], 95.00th=[  620],
     | 99.00th=[  892], 99.50th=[ 1064], 99.90th=[ 1416], 99.95th=[ 1784],
     | 99.99th=[ 5344]
    bw (KB  /s): min=    0, max=17648, per=25.01%, avg=16578.19, stdev=254.90
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.94%, 500=71.91%, 750=25.35%, 1000=1.15%
    lat (msec) : 2=0.60%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.32%, sys=4.44%, ctx=223082030, majf=0, minf=6804
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=119331841/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=429682MB, aggrb=61110KB/s, minb=61110KB/s, maxb=61110KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=416159MB, aggrb=59187KB/s, minb=59187KB/s, maxb=59187KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=479662MB, aggrb=68218KB/s, minb=68218KB/s, maxb=68218KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=466140MB, aggrb=66295KB/s, minb=66295KB/s, maxb=66295KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 05:24:02 AM EST
# 4kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 30866 IOPS | 123 MBps | 2590 μsec Average Latency 
SATA-R5-1: 30635 IOPS | 122 MBps | 2610 μsec Average Latency 
NVMe-R5-0: 6671 IOPS | 26 MBps | 2396 μsec Average Latency 
NVMe-R5-1: 6830 IOPS | 27 MBps | 2340 μsec Average Latency 

Totals
IOPS: 75002
MBPS: 298
Average Latency (μsec): 2484

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=11768: Fri Jan 15 07:25:03 2016
  read : io=868123MB, bw=123466KB/s, iops=30866, runt=7200002msec
    slat (usec): min=2, max=7940, avg=17.27, stdev=16.68
    clat (usec): min=118, max=1720.6K, avg=2572.90, stdev=1345.98
     lat (usec): min=207, max=1720.6K, avg=2590.36, stdev=1345.69
    clat percentiles (usec):
     |  1.00th=[ 1816],  5.00th=[ 2040], 10.00th=[ 2160], 20.00th=[ 2288],
     | 30.00th=[ 2352], 40.00th=[ 2448], 50.00th=[ 2512], 60.00th=[ 2608],
     | 70.00th=[ 2704], 80.00th=[ 2832], 90.00th=[ 3024], 95.00th=[ 3216],
     | 99.00th=[ 3760], 99.50th=[ 4080], 99.90th=[ 5600], 99.95th=[ 7072],
     | 99.99th=[10688]
    bw (KB  /s): min=    0, max=29360, per=20.01%, avg=24700.11, stdev=547.23
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=3.72%, 4=95.67%, 10=0.58%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.67%, sys=5.94%, ctx=374496374, majf=0, minf=20428
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=222239298/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=11773: Fri Jan 15 07:25:03 2016
  read : io=861628MB, bw=122543KB/s, iops=30635, runt=7200003msec
    slat (usec): min=1, max=1300, avg=10.68, stdev= 9.44
    clat (usec): min=177, max=1723.8K, avg=2599.40, stdev=1536.82
     lat (usec): min=218, max=1723.9K, avg=2610.22, stdev=1536.74
    clat percentiles (usec):
     |  1.00th=[ 1832],  5.00th=[ 2064], 10.00th=[ 2192], 20.00th=[ 2320],
     | 30.00th=[ 2384], 40.00th=[ 2480], 50.00th=[ 2544], 60.00th=[ 2640],
     | 70.00th=[ 2736], 80.00th=[ 2864], 90.00th=[ 3056], 95.00th=[ 3248],
     | 99.00th=[ 3728], 99.50th=[ 4048], 99.90th=[ 5472], 99.95th=[ 7008],
     | 99.99th=[10688]
    bw (KB  /s): min=    0, max=48928, per=20.01%, avg=24517.97, stdev=552.94
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=3.22%, 4=96.22%, 10=0.53%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.27%, sys=3.92%, ctx=377783919, majf=0, minf=20557
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=220576808/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=11778: Fri Jan 15 07:25:03 2016
  read : io=187623MB, bw=26684KB/s, iops=6671, runt=7200003msec
    slat (usec): min=2, max=4962, avg=20.03, stdev=18.38
    clat (usec): min=32, max=1736.9K, avg=2375.74, stdev=1334.95
     lat (usec): min=270, max=1736.9K, avg=2396.00, stdev=1334.73
    clat percentiles (usec):
     |  1.00th=[ 1608],  5.00th=[ 1848], 10.00th=[ 1944], 20.00th=[ 2064],
     | 30.00th=[ 2160], 40.00th=[ 2256], 50.00th=[ 2320], 60.00th=[ 2416],
     | 70.00th=[ 2512], 80.00th=[ 2640], 90.00th=[ 2864], 95.00th=[ 3024],
     | 99.00th=[ 3536], 99.50th=[ 3824], 99.90th=[ 5216], 99.95th=[ 6816],
     | 99.99th=[10432]
    bw (KB  /s): min=    0, max=10936, per=25.02%, avg=6676.44, stdev=159.58
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
    lat (usec) : 1000=0.03%
    lat (msec) : 2=13.68%, 4=85.94%, 10=0.33%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=0.74%, sys=1.99%, ctx=92458225, majf=0, minf=7934
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48031426/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=11782: Fri Jan 15 07:25:03 2016
  read : io=192105MB, bw=27322KB/s, iops=6830, runt=7200001msec
    slat (usec): min=1, max=2168, avg=10.23, stdev=10.03
    clat (usec): min=142, max=1703.1K, avg=2330.64, stdev=1456.09
     lat (usec): min=203, max=1703.1K, avg=2340.99, stdev=1456.04
    clat percentiles (usec):
     |  1.00th=[ 1544],  5.00th=[ 1784], 10.00th=[ 1880], 20.00th=[ 2024],
     | 30.00th=[ 2128], 40.00th=[ 2224], 50.00th=[ 2288], 60.00th=[ 2384],
     | 70.00th=[ 2480], 80.00th=[ 2608], 90.00th=[ 2800], 95.00th=[ 2992],
     | 99.00th=[ 3472], 99.50th=[ 3760], 99.90th=[ 5088], 99.95th=[ 6560],
     | 99.99th=[10304]
    bw (KB  /s): min=    0, max= 8392, per=25.02%, avg=6836.23, stdev=175.54
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.03%
    lat (msec) : 2=18.08%, 4=81.57%, 10=0.28%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=0.46%, sys=1.18%, ctx=88466892, majf=0, minf=7973
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49178791/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=868123MB, aggrb=123466KB/s, minb=123466KB/s, maxb=123466KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=861628MB, aggrb=122542KB/s, minb=122542KB/s, maxb=122542KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=187623MB, aggrb=26684KB/s, minb=26684KB/s, maxb=26684KB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=192105MB, aggrb=27321KB/s, minb=27321KB/s, maxb=27321KB/s, mint=7200001msec, maxt=7200001msec