client5
2016 January 13, 17:05
fio-2.1.10
CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
60 s ramp + 7200 s run, per test; s between tests

# 
# SR-71mach5 Configuration
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 

Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz   Status: Populated, Enabled Core Enabled: 10
Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz   Status: Populated, Enabled Core Enabled: 10
DIMM_A1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC441A  
DIMM_B1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC4159  
DIMM_C1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13A2C789  
DIMM_D1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13892F11  
DIMM_E1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC4378  
DIMM_F1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC4167  
DIMM_G1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC446A  
DIMM_H1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-CK0 | SN: 33AC446B  
____________________________________________________________________
MemTotal:       65701096 kB


# 
# actual mount paramters
# 

10.0.5.144:/control on /root/sr71/control type nfs4 (rw,relatime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=10.0.5.143,local_lock=none,addr=10.0.5.144)
10.10.8.100:/sr71/C0P5 on /root/sr71/SATA0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.8.10,local_lock=none,addr=10.10.8.100)
10.10.9.100:/sr71/C1P5 on /root/sr71/SATA1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.9.10,local_lock=none,addr=10.10.9.100)
10.10.8.100:/sr71/NVMeR5V0P5 on /root/sr71/NVMe0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.8.10,local_lock=none,addr=10.10.8.100)
10.10.9.100:/sr71/NVMeR5V1P5 on /root/sr71/NVMe1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.9.10,local_lock=none,addr=10.10.9.100)



# 
# fio job file
fio-2.1.10
# 

 seconds between fio tests
[global]
ioengine=libaio
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
group_reporting

[SATA-R5-01]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA0/fio.tst

[SATA-R5-11]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA1/fio.tst

[NVMe-R5-01]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe0/fio.tst

[NVMe-R5-11]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe1/fio.tst




# 
# 201601131705 test details
# 



# 
# Wed 13 Jan 2016 05:05:00 PM EST
# 8kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 8kB randwrite with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 3008 IOPS | 24 MBps | 329 μsec Average Latency 
SATA-R5-1: 2743 IOPS | 21 MBps | 361 μsec Average Latency 
NVMe-R5-0: 5048 IOPS | 40 MBps | 394 μsec Average Latency 
NVMe-R5-1: 5442 IOPS | 43 MBps | 365 μsec Average Latency 

Totals
IOPS: 16241
MBPS: 128
Average Latency (μsec): 362

SATA-R5-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=170318: Wed Jan 13 19:06:02 2016
  write: io=169206MB, bw=24065KB/s, iops=3008, runt=7200001msec
    slat (usec): min=3, max=2610, avg=18.71, stdev= 5.37
    clat (usec): min=2, max=199648, avg=310.32, stdev=321.89
     lat (usec): min=90, max=199658, avg=329.28, stdev=322.37
    clat percentiles (usec):
     |  1.00th=[  107],  5.00th=[  109], 10.00th=[  110], 20.00th=[  241],
     | 30.00th=[  346], 40.00th=[  346], 50.00th=[  350], 60.00th=[  350],
     | 70.00th=[  350], 80.00th=[  354], 90.00th=[  362], 95.00th=[  366],
     | 99.00th=[  390], 99.50th=[  410], 99.90th=[  494], 99.95th=[  620],
     | 99.99th=[ 1304]
    bw (KB  /s): min=    0, max=49648, per=100.00%, avg=24072.68, stdev=3879.39
    lat (usec) : 4=0.01%, 100=0.12%, 250=20.65%, 500=79.15%, 750=0.05%
    lat (usec) : 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=2.55%, sys=6.55%, ctx=43669245, majf=0, minf=1426
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=21658350/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=170319: Wed Jan 13 19:06:02 2016
  write: io=154307MB, bw=21946KB/s, iops=2743, runt=7200001msec
    slat (usec): min=3, max=173, avg=19.65, stdev= 5.37
    clat (usec): min=64, max=164015, avg=341.17, stdev=65.99
     lat (usec): min=94, max=164037, avg=361.11, stdev=66.03
    clat percentiles (usec):
     |  1.00th=[  110],  5.00th=[  241], 10.00th=[  322], 20.00th=[  346],
     | 30.00th=[  346], 40.00th=[  350], 50.00th=[  350], 60.00th=[  350],
     | 70.00th=[  350], 80.00th=[  354], 90.00th=[  366], 95.00th=[  370],
     | 99.00th=[  390], 99.50th=[  418], 99.90th=[  498], 99.95th=[  620],
     | 99.99th=[ 1128]
    bw (KB  /s): min=    0, max=40240, per=100.00%, avg=21952.57, stdev=1212.43
    lat (usec) : 100=0.04%, 250=5.81%, 500=94.05%, 750=0.06%, 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 250=0.01%
  cpu          : usr=2.62%, sys=6.76%, ctx=39852060, majf=0, minf=1438
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19751359/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=170320: Wed Jan 13 19:06:02 2016
  write: io=283975MB, bw=40388KB/s, iops=5048, runt=7200001msec
    slat (usec): min=2, max=165, avg=10.31, stdev= 4.32
    clat (usec): min=67, max=265111, avg=384.17, stdev=1849.49
     lat (usec): min=93, max=265116, avg=394.65, stdev=1849.47
    clat percentiles (usec):
     |  1.00th=[  187],  5.00th=[  239], 10.00th=[  330], 20.00th=[  358],
     | 30.00th=[  362], 40.00th=[  366], 50.00th=[  366], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  366], 95.00th=[  370],
     | 99.00th=[  390], 99.50th=[  414], 99.90th=[  652], 99.95th=[16768],
     | 99.99th=[95744]
    bw (KB  /s): min=    0, max=52080, per=100.00%, avg=40490.07, stdev=5439.71
    lat (usec) : 100=0.03%, 250=6.45%, 500=93.34%, 750=0.08%, 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.03%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=1.79%, sys=4.46%, ctx=72659268, majf=0, minf=1670
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=36348854/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=170321: Wed Jan 13 19:06:02 2016
  write: io=306161MB, bw=43543KB/s, iops=5442, runt=7200001msec
    slat (usec): min=2, max=572, avg=10.22, stdev= 4.27
    clat (usec): min=12, max=200187, avg=355.56, stdev=445.54
     lat (usec): min=91, max=200206, avg=365.94, stdev=445.43
    clat percentiles (usec):
     |  1.00th=[  187],  5.00th=[  241], 10.00th=[  342], 20.00th=[  358],
     | 30.00th=[  362], 40.00th=[  366], 50.00th=[  366], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  366], 95.00th=[  370],
     | 99.00th=[  390], 99.50th=[  406], 99.90th=[  516], 99.95th=[  692],
     | 99.99th=[ 1496]
    bw (KB  /s): min=    0, max=47424, per=100.00%, avg=43555.66, stdev=1737.08
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.03%, 250=5.98%, 500=93.86%
    lat (usec) : 750=0.08%, 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.87%, sys=4.82%, ctx=78306394, majf=0, minf=1823
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=39188579/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=169206MB, aggrb=24064KB/s, minb=24064KB/s, maxb=24064KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=154307MB, aggrb=21945KB/s, minb=21945KB/s, maxb=21945KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=283975MB, aggrb=40387KB/s, minb=40387KB/s, maxb=40387KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=306161MB, aggrb=43542KB/s, minb=43542KB/s, maxb=43542KB/s, mint=7200001msec, maxt=7200001msec



# 
# Wed 13 Jan 2016 07:06:03 PM EST
# 64kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 64kB randwrite with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1456 IOPS | 93 MBps | 683 μsec Average Latency 
SATA-R5-1: 1387 IOPS | 88 MBps | 717 μsec Average Latency 
NVMe-R5-0: 606 IOPS | 38 MBps | 13186 μsec Average Latency 
NVMe-R5-1: 666 IOPS | 42 MBps | 12008 μsec Average Latency 

Totals
IOPS: 4115
MBPS: 261
Average Latency (μsec): 6648

SATA-R5-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 10 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=175398: Wed Jan 13 21:07:07 2016
  write: io=655752MB, bw=93247KB/s, iops=1456, runt=7201181msec
    slat (usec): min=9, max=858, avg=43.13, stdev= 8.69
    clat (usec): min=115, max=3899.1K, avg=639.70, stdev=8559.49
     lat (usec): min=198, max=3900.3K, avg=683.17, stdev=8559.50
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  318], 10.00th=[  322], 20.00th=[  326],
     | 30.00th=[  326], 40.00th=[  330], 50.00th=[  334], 60.00th=[  346],
     | 70.00th=[  366], 80.00th=[  442], 90.00th=[  454], 95.00th=[  516],
     | 99.00th=[  612], 99.50th=[  876], 99.90th=[86528], 99.95th=[144384],
     | 99.99th=[350208]
    bw (KB  /s): min=    1, max=166656, per=100.00%, avg=96670.94, stdev=29650.70
    lat (usec) : 250=0.02%, 500=94.59%, 750=4.75%, 1000=0.20%
    lat (msec) : 2=0.11%, 4=0.03%, 10=0.02%, 20=0.01%, 50=0.11%
    lat (msec) : 100=0.06%, 250=0.07%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.71%, sys=4.31%, ctx=21173756, majf=0, minf=3255
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10492032/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=175399: Wed Jan 13 21:07:07 2016
  write: io=624699MB, bw=88828KB/s, iops=1387, runt=7201483msec
    slat (usec): min=10, max=1656, avg=42.87, stdev= 8.15
    clat (usec): min=156, max=3899.9K, avg=674.08, stdev=8769.72
     lat (usec): min=205, max=3899.1K, avg=717.32, stdev=8769.72
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  318], 10.00th=[  322], 20.00th=[  326],
     | 30.00th=[  326], 40.00th=[  330], 50.00th=[  334], 60.00th=[  346],
     | 70.00th=[  366], 80.00th=[  442], 90.00th=[  454], 95.00th=[  516],
     | 99.00th=[  612], 99.50th=[  932], 99.90th=[97792], 99.95th=[150528],
     | 99.99th=[350208]
    bw (KB  /s): min=    1, max=165888, per=100.00%, avg=92088.75, stdev=30189.74
    lat (usec) : 250=0.01%, 500=94.71%, 750=4.61%, 1000=0.21%
    lat (msec) : 2=0.11%, 4=0.02%, 10=0.01%, 20=0.02%, 50=0.12%
    lat (msec) : 100=0.08%, 250=0.08%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.63%, sys=4.13%, ctx=20181109, majf=0, minf=3337
  IO depths    : 1=100.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=9995186/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=175400: Wed Jan 13 21:07:07 2016
  write: io=273016MB, bw=38822KB/s, iops=606, runt=7201300msec
    slat (usec): min=8, max=1228, avg=39.97, stdev=21.01
    clat (usec): min=70, max=3980.1K, avg=13145.96, stdev=52955.75
     lat (usec): min=231, max=3980.1K, avg=13186.21, stdev=52955.45
    clat percentiles (usec):
     |  1.00th=[  306],  5.00th=[  362], 10.00th=[  402], 20.00th=[  450],
     | 30.00th=[  466], 40.00th=[  486], 50.00th=[  532], 60.00th=[  572],
     | 70.00th=[  604], 80.00th=[  692], 90.00th=[ 1064], 95.00th=[122368],
     | 99.00th=[199680], 99.50th=[276480], 99.90th=[464896], 99.95th=[610304],
     | 99.99th=[1011712]
    bw (KB  /s): min=    1, max=120291, per=25.49%, avg=9896.76, stdev=6945.98
    lat (usec) : 100=0.01%, 250=0.28%, 500=43.42%, 750=40.25%, 1000=5.42%
    lat (msec) : 2=2.69%, 4=0.28%, 10=0.11%, 20=0.05%, 50=0.58%
    lat (msec) : 100=1.51%, 250=4.80%, 500=0.52%, 750=0.06%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.11%, sys=0.28%, ctx=8022315, majf=0, minf=12266
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4368259/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=175404: Wed Jan 13 21:07:07 2016
  write: io=299790MB, bw=42630KB/s, iops=666, runt=7201225msec
    slat (usec): min=7, max=481, avg=31.94, stdev=13.34
    clat (usec): min=86, max=3931.1K, avg=11976.06, stdev=50108.98
     lat (usec): min=217, max=3931.1K, avg=12008.23, stdev=50108.84
    clat percentiles (usec):
     |  1.00th=[  314],  5.00th=[  350], 10.00th=[  398], 20.00th=[  446],
     | 30.00th=[  462], 40.00th=[  474], 50.00th=[  498], 60.00th=[  556],
     | 70.00th=[  588], 80.00th=[  660], 90.00th=[  988], 95.00th=[91648],
     | 99.00th=[199680], 99.50th=[257024], 99.90th=[448512], 99.95th=[585728],
     | 99.99th=[995328]
    bw (KB  /s): min=    1, max=255744, per=25.57%, avg=10899.49, stdev=7821.04
    lat (usec) : 100=0.01%, 250=0.09%, 500=50.53%, 750=34.87%, 1000=4.60%
    lat (msec) : 2=2.38%, 4=0.25%, 10=0.10%, 20=0.05%, 50=0.68%
    lat (msec) : 100=1.70%, 250=4.20%, 500=0.46%, 750=0.05%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.13%, sys=0.34%, ctx=8979221, majf=0, minf=19530
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4796631/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=655752MB, aggrb=93247KB/s, minb=93247KB/s, maxb=93247KB/s, mint=7201181msec, maxt=7201181msec

Run status group 1 (all jobs):
  WRITE: io=624699MB, aggrb=88827KB/s, minb=88827KB/s, maxb=88827KB/s, mint=7201483msec, maxt=7201483msec

Run status group 2 (all jobs):
  WRITE: io=273016MB, aggrb=38821KB/s, minb=38821KB/s, maxb=38821KB/s, mint=7201300msec, maxt=7201300msec

Run status group 3 (all jobs):
  WRITE: io=299790MB, aggrb=42629KB/s, minb=42629KB/s, maxb=42629KB/s, mint=7201225msec, maxt=7201225msec



# 
# Wed 13 Jan 2016 09:07:08 PM EST
# 8kB randrw with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randrw with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 5840 IOPS | 31 MBps | 7358 μsec Average Latency 
SATA-R5-1: 5730 IOPS | 30 MBps | 7500 μsec Average Latency 
NVMe-R5-0: 9748 IOPS | 52 MBps | 4366 μsec Average Latency 
NVMe-R5-1: 10349 IOPS | 55 MBps | 4091 μsec Average Latency 

Totals
IOPS: 31667
MBPS: 168
Average Latency (μsec): 5828

SATA-R5-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=180484: Wed Jan 13 23:08:10 2016
  read : io=220162MB, bw=31312KB/s, iops=3913, runt=7200004msec
    slat (usec): min=1, max=1410, avg=14.81, stdev=12.58
    clat (usec): min=21, max=1582.9K, avg=1807.15, stdev=4895.89
     lat (usec): min=127, max=1582.9K, avg=1822.13, stdev=4897.42
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  446], 10.00th=[  478], 20.00th=[  540],
     | 30.00th=[  596], 40.00th=[  628], 50.00th=[  684], 60.00th=[  748],
     | 70.00th=[  876], 80.00th=[ 1544], 90.00th=[ 3056], 95.00th=[ 9664],
     | 99.00th=[16768], 99.50th=[18304], 99.90th=[25216], 99.95th=[28032],
     | 99.99th=[83456]
    bw (KB  /s): min=    6, max=25840, per=25.30%, avg=7922.26, stdev=8365.86
  write: io=108437MB, bw=15422KB/s, iops=1927, runt=7200004msec
    slat (usec): min=2, max=1247, avg=19.78, stdev=15.25
    clat (usec): min=1, max=1749.5K, avg=12875.14, stdev=33553.19
     lat (usec): min=134, max=1749.5K, avg=12895.10, stdev=33555.03
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  572], 10.00th=[ 1560], 20.00th=[ 2896],
     | 30.00th=[ 3504], 40.00th=[ 4016], 50.00th=[ 4512], 60.00th=[ 5088],
     | 70.00th=[ 5920], 80.00th=[ 7648], 90.00th=[17792], 95.00th=[64256],
     | 99.00th=[187392], 99.50th=[209920], 99.90th=[261120], 99.95th=[301056],
     | 99.99th=[659456]
    bw (KB  /s): min=    0, max=12576, per=25.30%, avg=3902.02, stdev=4119.51
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.05%, 500=10.45%
    lat (usec) : 750=32.51%, 1000=9.23%
    lat (msec) : 2=7.00%, 4=15.90%, 10=16.94%, 20=4.53%, 50=1.50%
    lat (msec) : 100=0.70%, 250=1.13%, 500=0.04%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.45%, sys=1.29%, ctx=72886895, majf=0, minf=13709
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=28180682/w=13879869/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=180488: Wed Jan 13 23:08:10 2016
  read : io=215998MB, bw=30720KB/s, iops=3839, runt=7200011msec
    slat (usec): min=1, max=1002, avg=14.96, stdev=12.72
    clat (usec): min=37, max=1982.2K, avg=1841.99, stdev=5120.04
     lat (usec): min=135, max=1982.2K, avg=1857.12, stdev=5121.55
    clat percentiles (usec):
     |  1.00th=[  362],  5.00th=[  446], 10.00th=[  478], 20.00th=[  540],
     | 30.00th=[  596], 40.00th=[  628], 50.00th=[  684], 60.00th=[  748],
     | 70.00th=[  892], 80.00th=[ 1640], 90.00th=[ 3120], 95.00th=[ 9920],
     | 99.00th=[17280], 99.50th=[18816], 99.90th=[25472], 99.95th=[28032],
     | 99.99th=[57600]
    bw (KB  /s): min=    0, max=28224, per=25.30%, avg=7770.69, stdev=8320.83
  write: io=106390MB, bw=15131KB/s, iops=1891, runt=7200011msec
    slat (usec): min=2, max=1213, avg=20.00, stdev=15.55
    clat (usec): min=43, max=1986.4K, avg=13123.40, stdev=34338.15
     lat (usec): min=140, max=1986.4K, avg=13143.59, stdev=34340.02
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  572], 10.00th=[ 1544], 20.00th=[ 2864],
     | 30.00th=[ 3472], 40.00th=[ 3952], 50.00th=[ 4448], 60.00th=[ 5024],
     | 70.00th=[ 5920], 80.00th=[ 7840], 90.00th=[18560], 95.00th=[65280],
     | 99.00th=[191488], 99.50th=[216064], 99.90th=[264192], 99.95th=[296960],
     | 99.99th=[675840]
    bw (KB  /s): min=    0, max=13824, per=25.30%, avg=3827.47, stdev=4099.31
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.05%, 500=10.34%, 750=32.45%
    lat (usec) : 1000=9.04%
    lat (msec) : 2=7.07%, 4=16.44%, 10=16.50%, 20=4.64%, 50=1.57%
    lat (msec) : 100=0.69%, 250=1.16%, 500=0.04%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.44%, sys=1.28%, ctx=71425106, majf=0, minf=14643
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=27647737/w=13617834/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=180492: Wed Jan 13 23:08:10 2016
  read : io=367412MB, bw=52254KB/s, iops=6531, runt=7200007msec
    slat (usec): min=1, max=1348, avg=14.13, stdev=11.46
    clat (usec): min=2, max=1596.6K, avg=1157.60, stdev=3494.62
     lat (usec): min=97, max=1596.7K, avg=1171.89, stdev=3495.25
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  382], 10.00th=[  446], 20.00th=[  486],
     | 30.00th=[  524], 40.00th=[  588], 50.00th=[  628], 60.00th=[  740],
     | 70.00th=[ 1032], 80.00th=[ 1864], 90.00th=[ 2640], 95.00th=[ 3344],
     | 99.00th=[ 4832], 99.50th=[ 5408], 99.90th=[ 7456], 99.95th=[ 8384],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=35440, per=25.14%, avg=13135.28, stdev=9314.22
  write: io=180979MB, bw=25739KB/s, iops=3217, runt=7200007msec
    slat (usec): min=2, max=509, avg=18.40, stdev=13.54
    clat (usec): min=18, max=1602.9K, avg=7543.06, stdev=11990.29
     lat (usec): min=123, max=1602.1K, avg=7561.63, stdev=11992.18
    clat percentiles (usec):
     |  1.00th=[  338],  5.00th=[  498], 10.00th=[  996], 20.00th=[ 1992],
     | 30.00th=[ 2448], 40.00th=[ 2832], 50.00th=[ 3216], 60.00th=[ 3888],
     | 70.00th=[ 7072], 80.00th=[14016], 90.00th=[21376], 95.00th=[26240],
     | 99.00th=[34560], 99.50th=[37632], 99.90th=[44800], 99.95th=[51456],
     | 99.99th=[481280]
    bw (KB  /s): min=    0, max=17344, per=25.14%, avg=6470.07, stdev=4583.62
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.21%
    lat (usec) : 500=18.17%, 750=25.55%, 1000=5.85%
    lat (msec) : 2=11.54%, 4=24.01%, 10=6.16%, 20=4.62%, 50=3.86%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.79%, sys=2.23%, ctx=121178587, majf=0, minf=14159
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47028789/w=23165304/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=180496: Wed Jan 13 23:08:10 2016
  read : io=390051MB, bw=55474KB/s, iops=6934, runt=7200009msec
    slat (usec): min=2, max=535, avg=15.19, stdev=12.39
    clat (usec): min=12, max=1623.8K, avg=1130.19, stdev=3548.20
     lat (usec): min=97, max=1623.8K, avg=1145.55, stdev=3548.77
    clat percentiles (usec):
     |  1.00th=[  314],  5.00th=[  378], 10.00th=[  442], 20.00th=[  486],
     | 30.00th=[  524], 40.00th=[  588], 50.00th=[  636], 60.00th=[  740],
     | 70.00th=[ 1048], 80.00th=[ 1832], 90.00th=[ 2544], 95.00th=[ 3088],
     | 99.00th=[ 4576], 99.50th=[ 5024], 99.90th=[ 7200], 99.95th=[ 8160],
     | 99.99th=[12096]
    bw (KB  /s): min=    0, max=36096, per=25.13%, avg=13937.92, stdev=9011.08
  write: io=192149MB, bw=27328KB/s, iops=3415, runt=7200009msec
    slat (usec): min=2, max=578, avg=19.98, stdev=14.79
    clat (usec): min=1, max=1627.9K, avg=7016.88, stdev=10677.27
     lat (usec): min=116, max=1627.9K, avg=7037.04, stdev=10679.07
    clat percentiles (usec):
     |  1.00th=[  330],  5.00th=[  498], 10.00th=[ 1064], 20.00th=[ 2064],
     | 30.00th=[ 2512], 40.00th=[ 2928], 50.00th=[ 3376], 60.00th=[ 4192],
     | 70.00th=[ 7520], 80.00th=[12736], 90.00th=[18560], 95.00th=[22656],
     | 99.00th=[29824], 99.50th=[32384], 99.90th=[38656], 99.95th=[44288],
     | 99.99th=[411648]
    bw (KB  /s): min=    0, max=17536, per=25.13%, avg=6866.47, stdev=4436.84
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.24%
    lat (usec) : 500=18.19%, 750=24.99%, 1000=6.04%
    lat (msec) : 2=11.84%, 4=23.65%, 10=6.73%, 20=5.69%, 50=2.63%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.83%, sys=2.42%, ctx=128102243, majf=0, minf=14419
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49926477/w=24595049/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=220162MB, aggrb=31311KB/s, minb=31311KB/s, maxb=31311KB/s, mint=7200004msec, maxt=7200004msec
  WRITE: io=108437MB, aggrb=15422KB/s, minb=15422KB/s, maxb=15422KB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=215998MB, aggrb=30719KB/s, minb=30719KB/s, maxb=30719KB/s, mint=7200011msec, maxt=7200011msec
  WRITE: io=106390MB, aggrb=15130KB/s, minb=15130KB/s, maxb=15130KB/s, mint=7200011msec, maxt=7200011msec

Run status group 2 (all jobs):
   READ: io=367412MB, aggrb=52254KB/s, minb=52254KB/s, maxb=52254KB/s, mint=7200007msec, maxt=7200007msec
  WRITE: io=180979MB, aggrb=25739KB/s, minb=25739KB/s, maxb=25739KB/s, mint=7200007msec, maxt=7200007msec

Run status group 3 (all jobs):
   READ: io=390051MB, aggrb=55473KB/s, minb=55473KB/s, maxb=55473KB/s, mint=7200009msec, maxt=7200009msec
  WRITE: io=192149MB, aggrb=27327KB/s, minb=27327KB/s, maxb=27327KB/s, mint=7200009msec, maxt=7200009msec



# 
# Wed 13 Jan 2016 11:08:12 PM EST
# 64kB randrw with SATA 5 jobs and 4 I/Os Outstanding
# 64kB randrw with NVMe 5 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1848 IOPS | 79 MBps | 14658 μsec Average Latency 
SATA-R5-1: 1793 IOPS | 76 MBps | 15113 μsec Average Latency 
NVMe-R5-0: 1523 IOPS | 65 MBps | 7890 μsec Average Latency 
NVMe-R5-1: 1626 IOPS | 69 MBps | 7382 μsec Average Latency 

Totals
IOPS: 6790
MBPS: 289
Average Latency (μsec): 11260

SATA-R5-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=185698: Thu Jan 14 01:09:13 2016
  read : io=557829MB, bw=79335KB/s, iops=1239, runt=7200029msec
    slat (usec): min=5, max=679, avg=33.12, stdev=20.59
    clat (usec): min=161, max=4137.9K, avg=3326.37, stdev=30167.07
     lat (usec): min=215, max=4137.9K, avg=3359.75, stdev=30167.19
    clat percentiles (usec):
     |  1.00th=[  732],  5.00th=[  884], 10.00th=[  972], 20.00th=[ 1080],
     | 30.00th=[ 1176], 40.00th=[ 1256], 50.00th=[ 1336], 60.00th=[ 1448],
     | 70.00th=[ 1624], 80.00th=[ 2544], 90.00th=[ 5024], 95.00th=[ 7776],
     | 99.00th=[17792], 99.50th=[42752], 99.90th=[201728], 99.95th=[552960],
     | 99.99th=[1548288]
    bw (KB  /s): min=    1, max=55040, per=23.25%, avg=18448.10, stdev=12540.33
  write: io=274385MB, bw=39023KB/s, iops=609, runt=7200029msec
    slat (usec): min=9, max=555, avg=54.94, stdev=27.09
    clat (usec): min=112, max=4284.1K, avg=25903.09, stdev=99844.03
     lat (usec): min=226, max=4285.3K, avg=25958.33, stdev=99844.07
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[ 1128], 10.00th=[ 5280], 20.00th=[ 7456],
     | 30.00th=[ 8640], 40.00th=[ 9536], 50.00th=[10432], 60.00th=[11584],
     | 70.00th=[13760], 80.00th=[18560], 90.00th=[47872], 95.00th=[70144],
     | 99.00th=[276480], 99.50th=[528384], 99.90th=[1630208], 99.95th=[2007040],
     | 99.99th=[2834432]
    bw (KB  /s): min=    1, max=23168, per=23.26%, avg=9077.38, stdev=6106.55
    lat (usec) : 250=0.01%, 500=0.37%, 750=1.18%, 1000=7.93%
    lat (msec) : 2=44.39%, 4=7.31%, 10=18.73%, 20=13.46%, 50=3.19%
    lat (msec) : 100=2.15%, 250=0.86%, 500=0.21%, 750=0.06%, 1000=0.04%
    lat (msec) : 2000=0.08%, >=2000=0.02%
  cpu          : usr=0.38%, sys=0.82%, ctx=25486554, majf=0, minf=45867
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8925269/w=4390137/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=185703: Thu Jan 14 01:09:13 2016
  read : io=541068MB, bw=76952KB/s, iops=1202, runt=7200022msec
    slat (usec): min=5, max=749, avg=33.39, stdev=20.96
    clat (usec): min=174, max=4214.3K, avg=3429.24, stdev=31679.24
     lat (usec): min=211, max=4214.3K, avg=3462.93, stdev=31679.36
    clat percentiles (usec):
     |  1.00th=[  716],  5.00th=[  860], 10.00th=[  948], 20.00th=[ 1064],
     | 30.00th=[ 1144], 40.00th=[ 1240], 50.00th=[ 1336], 60.00th=[ 1464],
     | 70.00th=[ 1672], 80.00th=[ 2864], 90.00th=[ 5408], 95.00th=[ 8096],
     | 99.00th=[16512], 99.50th=[41728], 99.90th=[203776], 99.95th=[610304],
     | 99.99th=[1630208]
    bw (KB  /s): min=    1, max=52166, per=23.33%, avg=17953.48, stdev=12429.69
  write: io=266147MB, bw=37852KB/s, iops=591, runt=7200022msec
    slat (usec): min=8, max=535, avg=54.83, stdev=27.18
    clat (usec): min=62, max=4275.9K, avg=26709.31, stdev=101849.66
     lat (usec): min=238, max=4275.1K, avg=26764.49, stdev=101849.72
    clat percentiles (usec):
     |  1.00th=[  474],  5.00th=[ 1080], 10.00th=[ 4960], 20.00th=[ 7328],
     | 30.00th=[ 8512], 40.00th=[ 9536], 50.00th=[10432], 60.00th=[11712],
     | 70.00th=[14272], 80.00th=[20096], 90.00th=[51968], 95.00th=[72192],
     | 99.00th=[276480], 99.50th=[552960], 99.90th=[1630208], 99.95th=[2023424],
     | 99.99th=[2899968]
    bw (KB  /s): min=    1, max=24320, per=23.33%, avg=8830.94, stdev=6066.21
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.40%, 750=1.42%, 1000=9.15%
    lat (msec) : 2=41.91%, 4=7.65%, 10=19.52%, 20=12.77%, 50=3.46%
    lat (msec) : 100=2.46%, 250=0.85%, 500=0.21%, 750=0.06%, 1000=0.04%
    lat (msec) : 2000=0.09%, >=2000=0.02%
  cpu          : usr=0.36%, sys=0.80%, ctx=24763771, majf=0, minf=48178
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8657092/w=4258329/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=185708: Thu Jan 14 01:09:13 2016
  read : io=459591MB, bw=65364KB/s, iops=1021, runt=7200002msec
    slat (usec): min=5, max=931, avg=30.07, stdev=18.24
    clat (usec): min=1, max=4339.7K, avg=3938.30, stdev=35708.39
     lat (usec): min=175, max=4339.1K, avg=3968.61, stdev=35708.50
    clat percentiles (usec):
     |  1.00th=[  548],  5.00th=[  692], 10.00th=[  788], 20.00th=[  964],
     | 30.00th=[ 1208], 40.00th=[ 1816], 50.00th=[ 2544], 60.00th=[ 3024],
     | 70.00th=[ 3600], 80.00th=[ 4320], 90.00th=[ 5088], 95.00th=[ 5856],
     | 99.00th=[ 8384], 99.50th=[10816], 99.90th=[264192], 99.95th=[798720],
     | 99.99th=[1794048]
    bw (KB  /s): min=    1, max=66688, per=23.50%, avg=15357.79, stdev=10202.89
  write: io=226328MB, bw=32189KB/s, iops=502, runt=7200002msec
    slat (usec): min=9, max=607, avg=49.61, stdev=24.94
    clat (usec): min=25, max=4348.7K, avg=11763.60, stdev=67727.79
     lat (usec): min=217, max=4348.4K, avg=11813.49, stdev=67727.81
    clat percentiles (usec):
     |  1.00th=[  422],  5.00th=[  756], 10.00th=[ 1032], 20.00th=[ 1768],
     | 30.00th=[ 2640], 40.00th=[ 4448], 50.00th=[ 7008], 60.00th=[ 9280],
     | 70.00th=[11200], 80.00th=[13120], 90.00th=[15552], 95.00th=[17536],
     | 99.00th=[93696], 99.50th=[173056], 99.90th=[1204224], 99.95th=[1597440],
     | 99.99th=[2473984]
    bw (KB  /s): min=    1, max=32640, per=23.50%, avg=7564.09, stdev=5027.01
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=0.96%
    lat (usec) : 750=5.94%, 1000=10.99%
    lat (msec) : 2=17.94%, 4=27.69%, 10=24.13%, 20=11.27%, 50=0.44%
    lat (msec) : 100=0.11%, 250=0.33%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.06%, >=2000=0.01%
  cpu          : usr=0.28%, sys=0.69%, ctx=21479496, majf=0, minf=23004
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7353453/w=3621239/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=185713: Thu Jan 14 01:09:13 2016
  read : io=490655MB, bw=69782KB/s, iops=1090, runt=7200011msec
    slat (usec): min=5, max=792, avg=33.87, stdev=20.73
    clat (usec): min=22, max=4153.8K, avg=3703.77, stdev=34517.82
     lat (usec): min=189, max=4153.9K, avg=3737.93, stdev=34517.91
    clat percentiles (usec):
     |  1.00th=[  548],  5.00th=[  700], 10.00th=[  796], 20.00th=[  972],
     | 30.00th=[ 1208], 40.00th=[ 1688], 50.00th=[ 2352], 60.00th=[ 2832],
     | 70.00th=[ 3280], 80.00th=[ 3952], 90.00th=[ 4768], 95.00th=[ 5408],
     | 99.00th=[ 7840], 99.50th=[10048], 99.90th=[238592], 99.95th=[724992],
     | 99.99th=[1728512]
    bw (KB  /s): min=    1, max=70656, per=23.49%, avg=16391.57, stdev=10171.85
  write: io=241478MB, bw=34344KB/s, iops=536, runt=7200011msec
    slat (usec): min=9, max=716, avg=54.77, stdev=26.95
    clat (usec): min=18, max=4155.8K, avg=10972.80, stdev=65829.50
     lat (usec): min=221, max=4155.9K, avg=11027.90, stdev=65829.46
    clat percentiles (usec):
     |  1.00th=[  418],  5.00th=[  756], 10.00th=[ 1048], 20.00th=[ 1800],
     | 30.00th=[ 2640], 40.00th=[ 4192], 50.00th=[ 6368], 60.00th=[ 8384],
     | 70.00th=[10176], 80.00th=[11968], 90.00th=[14144], 95.00th=[16064],
     | 99.00th=[84480], 99.50th=[156672], 99.90th=[1138688], 99.95th=[1581056],
     | 99.99th=[2506752]
    bw (KB  /s): min=    1, max=36864, per=23.50%, avg=8070.48, stdev=5006.12
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=1.00%
    lat (usec) : 750=5.63%, 1000=10.74%
    lat (msec) : 2=19.87%, 4=29.77%, 10=22.43%, 20=9.71%, 50=0.22%
    lat (msec) : 100=0.10%, 250=0.32%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.05%, >=2000=0.01%
  cpu          : usr=0.34%, sys=0.74%, ctx=22830626, majf=0, minf=24769
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7850472/w=3863646/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=557829MB, aggrb=79335KB/s, minb=79335KB/s, maxb=79335KB/s, mint=7200029msec, maxt=7200029msec
  WRITE: io=274385MB, aggrb=39023KB/s, minb=39023KB/s, maxb=39023KB/s, mint=7200029msec, maxt=7200029msec

Run status group 1 (all jobs):
   READ: io=541068MB, aggrb=76951KB/s, minb=76951KB/s, maxb=76951KB/s, mint=7200022msec, maxt=7200022msec
  WRITE: io=266147MB, aggrb=37851KB/s, minb=37851KB/s, maxb=37851KB/s, mint=7200022msec, maxt=7200022msec

Run status group 2 (all jobs):
   READ: io=459591MB, aggrb=65364KB/s, minb=65364KB/s, maxb=65364KB/s, mint=7200002msec, maxt=7200002msec
  WRITE: io=226328MB, aggrb=32188KB/s, minb=32188KB/s, maxb=32188KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=490655MB, aggrb=69781KB/s, minb=69781KB/s, maxb=69781KB/s, mint=7200011msec, maxt=7200011msec
  WRITE: io=241478MB, aggrb=34343KB/s, minb=34343KB/s, maxb=34343KB/s, mint=7200011msec, maxt=7200011msec



# 
# Thu 14 Jan 2016 01:09:15 AM EST
# 1mB write with SATA 1 jobs and 2 I/Os Outstanding
# 1mB write with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 338 IOPS | 347 MBps | 5900 μsec Average Latency 
SATA-R5-1: 310 IOPS | 318 MBps | 6430 μsec Average Latency 
NVMe-R5-0: 140 IOPS | 143 MBps | 57040 μsec Average Latency 
NVMe-R5-1: 136 IOPS | 139 MBps | 58630 μsec Average Latency 

Totals
IOPS: 924
MBPS: 947
Average Latency (μsec): 32000

SATA-R5-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
SATA-R5-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-01: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=191608: Thu Jan 14 03:10:17 2016
  write: io=2383.2GB, bw=347066KB/s, iops=338, runt=7200093msec
    slat (usec): min=94, max=995, avg=188.71, stdev=38.65
    clat (msec): min=1, max=549, avg= 5.71, stdev=14.23
     lat (msec): min=1, max=549, avg= 5.90, stdev=14.23
    clat percentiles (usec):
     |  1.00th=[ 1352],  5.00th=[ 1576], 10.00th=[ 1832], 20.00th=[ 1928],
     | 30.00th=[ 1992], 40.00th=[ 2040], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2416], 80.00th=[ 2992], 90.00th=[ 6816], 95.00th=[37120],
     | 99.00th=[68096], 99.50th=[83456], 99.90th=[138240], 99.95th=[183296],
     | 99.99th=[268288]
    bw (KB  /s): min=   17, max=854016, per=100.00%, avg=349382.29, stdev=118587.89
    lat (msec) : 2=33.06%, 4=52.50%, 10=7.92%, 20=0.70%, 50=2.88%
    lat (msec) : 100=2.66%, 250=0.26%, 500=0.01%, 750=0.01%
  cpu          : usr=1.42%, sys=2.94%, ctx=11725480, majf=0, minf=67241
  IO depths    : 1=0.1%, 2=100.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2440341/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=191609: Thu Jan 14 03:10:17 2016
  write: io=2185.2GB, bw=318240KB/s, iops=310, runt=7200031msec
    slat (usec): min=95, max=2188, avg=233.62, stdev=54.63
    clat (msec): min=1, max=604, avg= 6.20, stdev=15.53
     lat (msec): min=1, max=605, avg= 6.43, stdev=15.53
    clat percentiles (usec):
     |  1.00th=[ 1400],  5.00th=[ 1688], 10.00th=[ 1832], 20.00th=[ 1928],
     | 30.00th=[ 1992], 40.00th=[ 2040], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2384], 80.00th=[ 2960], 90.00th=[ 7392], 95.00th=[41216],
     | 99.00th=[74240], 99.50th=[89600], 99.90th=[146432], 99.95th=[199680],
     | 99.99th=[292864]
    bw (KB  /s): min=   17, max=827784, per=100.00%, avg=320264.09, stdev=109256.66
    lat (msec) : 2=31.36%, 4=53.65%, 10=7.77%, 20=0.76%, 50=2.86%
    lat (msec) : 100=3.25%, 250=0.32%, 500=0.02%, 750=0.01%
  cpu          : usr=1.48%, sys=2.99%, ctx=10535981, majf=0, minf=66200
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2237634/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=191610: Thu Jan 14 03:10:17 2016
  write: io=986.11GB, bw=143612KB/s, iops=140, runt=7200013msec
    slat (usec): min=95, max=2265, avg=269.89, stdev=67.01
    clat (msec): min=1, max=1368, avg=56.77, stdev=72.76
     lat (msec): min=1, max=1368, avg=57.04, stdev=72.76
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   11], 60.00th=[   56],
     | 70.00th=[   76], 80.00th=[  106], 90.00th=[  157], 95.00th=[  200],
     | 99.00th=[  293], 99.50th=[  379], 99.90th=[  570], 99.95th=[  635],
     | 99.99th=[  865]
    bw (KB  /s): min=   17, max=813477, per=100.00%, avg=145077.05, stdev=54874.58
    lat (msec) : 2=0.65%, 4=7.33%, 10=41.52%, 20=4.65%, 50=3.93%
    lat (msec) : 100=20.56%, 250=19.60%, 500=1.62%, 750=0.11%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.77%, sys=1.58%, ctx=4643429, majf=0, minf=65660
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.5%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=1009765/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=191611: Thu Jan 14 03:10:17 2016
  write: io=982414MB, bw=139720KB/s, iops=136, runt=7200071msec
    slat (usec): min=97, max=1575, avg=272.89, stdev=62.73
    clat (msec): min=1, max=1381, avg=58.36, stdev=73.66
     lat (msec): min=1, max=1382, avg=58.63, stdev=73.66
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   11], 60.00th=[   58],
     | 70.00th=[   79], 80.00th=[  111], 90.00th=[  161], 95.00th=[  204],
     | 99.00th=[  297], 99.50th=[  375], 99.90th=[  545], 99.95th=[  619],
     | 99.99th=[  848]
    bw (KB  /s): min=   17, max=720667, per=100.00%, avg=140983.32, stdev=51470.56
    lat (msec) : 2=0.73%, 4=7.61%, 10=40.48%, 20=4.72%, 50=3.83%
    lat (msec) : 100=20.30%, 250=20.45%, 500=1.76%, 750=0.11%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.76%, sys=1.55%, ctx=4508415, majf=0, minf=66186
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=982407/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
  WRITE: io=2383.2GB, aggrb=347066KB/s, minb=347066KB/s, maxb=347066KB/s, mint=7200093msec, maxt=7200093msec

Run status group 1 (all jobs):
  WRITE: io=2185.2GB, aggrb=318240KB/s, minb=318240KB/s, maxb=318240KB/s, mint=7200031msec, maxt=7200031msec

Run status group 2 (all jobs):
  WRITE: io=986.11GB, aggrb=143611KB/s, minb=143611KB/s, maxb=143611KB/s, mint=7200013msec, maxt=7200013msec

Run status group 3 (all jobs):
  WRITE: io=982414MB, aggrb=139719KB/s, minb=139719KB/s, maxb=139719KB/s, mint=7200071msec, maxt=7200071msec



# 
# Thu 14 Jan 2016 03:10:18 AM EST
# 1mB read with SATA 1 jobs and 8 I/Os Outstanding
# 1mB read with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 728 IOPS | 745 MBps | 10990 μsec Average Latency 
SATA-R5-1: 729 IOPS | 746 MBps | 10960 μsec Average Latency 
NVMe-R5-0: 348 IOPS | 357 MBps | 5730 μsec Average Latency 
NVMe-R5-1: 350 IOPS | 359 MBps | 5700 μsec Average Latency 

Totals
IOPS: 2155
MBPS: 2207
Average Latency (μsec): 8345

SATA-R5-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=196680: Thu Jan 14 05:11:20 2016
  read : io=5118.1GB, bw=745490KB/s, iops=728, runt=7200007msec
    slat (usec): min=66, max=10444, avg=178.60, stdev=88.54
    clat (msec): min=1, max=218, avg=10.81, stdev= 2.16
     lat (msec): min=1, max=218, avg=10.99, stdev= 2.16
    clat percentiles (usec):
     |  1.00th=[ 5984],  5.00th=[ 8032], 10.00th=[ 8768], 20.00th=[ 9536],
     | 30.00th=[ 9920], 40.00th=[10176], 50.00th=[10560], 60.00th=[10816],
     | 70.00th=[11072], 80.00th=[11840], 90.00th=[13376], 95.00th=[15168],
     | 99.00th=[18304], 99.50th=[19328], 99.90th=[21632], 99.95th=[22912],
     | 99.99th=[25984]
    bw (KB  /s): min=   17, max=899072, per=100.00%, avg=746111.49, stdev=26266.01
    lat (msec) : 2=0.01%, 4=0.06%, 10=32.77%, 20=66.90%, 50=0.28%
    lat (msec) : 250=0.01%
  cpu          : usr=0.35%, sys=8.52%, ctx=26021835, majf=0, minf=69983
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5241726/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=196681: Thu Jan 14 05:11:20 2016
  read : io=5129.3GB, bw=746998KB/s, iops=729, runt=7200008msec
    slat (usec): min=70, max=8209, avg=239.23, stdev=82.33
    clat (msec): min=1, max=35, avg=10.72, stdev= 2.17
     lat (msec): min=1, max=35, avg=10.96, stdev= 2.17
    clat percentiles (usec):
     |  1.00th=[ 6048],  5.00th=[ 7904], 10.00th=[ 8640], 20.00th=[ 9408],
     | 30.00th=[ 9792], 40.00th=[10048], 50.00th=[10432], 60.00th=[10688],
     | 70.00th=[11072], 80.00th=[11712], 90.00th=[13376], 95.00th=[15168],
     | 99.00th=[18304], 99.50th=[19072], 99.90th=[21632], 99.95th=[22912],
     | 99.99th=[25728]
    bw (KB  /s): min=   17, max=899072, per=100.00%, avg=747620.60, stdev=27856.97
    lat (msec) : 2=0.01%, 4=0.05%, 10=36.92%, 20=62.76%, 50=0.27%
  cpu          : usr=0.63%, sys=8.54%, ctx=25623913, majf=0, minf=68934
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5252330/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=196682: Thu Jan 14 05:11:20 2016
  read : io=2453.5GB, bw=357306KB/s, iops=348, runt=7200005msec
    slat (usec): min=66, max=10162, avg=160.16, stdev=83.42
    clat (usec): min=601, max=217157, avg=5568.46, stdev=2065.59
     lat (msec): min=1, max=217, avg= 5.73, stdev= 2.07
    clat percentiles (usec):
     |  1.00th=[ 1608],  5.00th=[ 2608], 10.00th=[ 3568], 20.00th=[ 4384],
     | 30.00th=[ 4768], 40.00th=[ 5088], 50.00th=[ 5280], 60.00th=[ 5536],
     | 70.00th=[ 5856], 80.00th=[ 6368], 90.00th=[ 7648], 95.00th=[10048],
     | 99.00th=[13120], 99.50th=[14016], 99.90th=[15808], 99.95th=[16768],
     | 99.99th=[19584]
    bw (KB  /s): min=   17, max=425133, per=100.00%, avg=357634.15, stdev=18154.41
    lat (usec) : 750=0.01%, 1000=0.01%
    lat (msec) : 2=2.52%, 4=11.67%, 10=80.80%, 20=5.01%, 50=0.01%
    lat (msec) : 250=0.01%
  cpu          : usr=0.20%, sys=3.52%, ctx=12668391, majf=0, minf=67903
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2512306/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=196683: Thu Jan 14 05:11:20 2016
  read : io=2467.6GB, bw=359357KB/s, iops=350, runt=7200008msec
    slat (usec): min=70, max=10529, avg=221.94, stdev=133.52
    clat (usec): min=143, max=26249, avg=5473.88, stdev=2055.18
     lat (msec): min=1, max=26, avg= 5.70, stdev= 2.06
    clat percentiles (usec):
     |  1.00th=[ 1640],  5.00th=[ 2608], 10.00th=[ 3504], 20.00th=[ 4256],
     | 30.00th=[ 4640], 40.00th=[ 4960], 50.00th=[ 5216], 60.00th=[ 5472],
     | 70.00th=[ 5792], 80.00th=[ 6240], 90.00th=[ 7584], 95.00th=[ 9920],
     | 99.00th=[12992], 99.50th=[13888], 99.90th=[15808], 99.95th=[16768],
     | 99.99th=[19328]
    bw (KB  /s): min=   17, max=425133, per=100.00%, avg=359686.18, stdev=18992.24
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=2.43%, 4=13.29%, 10=79.38%, 20=4.90%, 50=0.01%
  cpu          : usr=0.18%, sys=3.57%, ctx=12770035, majf=0, minf=67510
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2526733/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=5118.1GB, aggrb=745490KB/s, minb=745490KB/s, maxb=745490KB/s, mint=7200007msec, maxt=7200007msec

Run status group 1 (all jobs):
   READ: io=5129.3GB, aggrb=746998KB/s, minb=746998KB/s, maxb=746998KB/s, mint=7200008msec, maxt=7200008msec

Run status group 2 (all jobs):
   READ: io=2453.5GB, aggrb=357305KB/s, minb=357305KB/s, maxb=357305KB/s, mint=7200005msec, maxt=7200005msec

Run status group 3 (all jobs):
   READ: io=2467.6GB, aggrb=359357KB/s, minb=359357KB/s, maxb=359357KB/s, mint=7200008msec, maxt=7200008msec



# 
# Thu 14 Jan 2016 05:11:22 AM EST
# 256kB write with SATA 1 jobs and 8 I/Os Outstanding
# 256kB write with NVMe 1 jobs and 1 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1430 IOPS | 366 MBps | 5590 μsec Average Latency 
SATA-R5-1: 1407 IOPS | 360 MBps | 5681 μsec Average Latency 
NVMe-R5-0: 503 IOPS | 128 MBps | 1983 μsec Average Latency 
NVMe-R5-1: 497 IOPS | 127 MBps | 2008 μsec Average Latency 

Totals
IOPS: 3837
MBPS: 981
Average Latency (μsec): 3815

SATA-R5-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
NVMe-R5-11: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=201808: Thu Jan 14 07:12:23 2016
  write: io=2514.8GB, bw=366230KB/s, iops=1430, runt=7200002msec
    slat (usec): min=24, max=290, avg=46.17, stdev= 9.88
    clat (usec): min=408, max=526892, avg=5544.33, stdev=15228.69
     lat (usec): min=498, max=526936, avg=5590.69, stdev=15228.47
    clat percentiles (usec):
     |  1.00th=[  708],  5.00th=[  956], 10.00th=[ 1192], 20.00th=[ 1448],
     | 30.00th=[ 1592], 40.00th=[ 1704], 50.00th=[ 1816], 60.00th=[ 1832],
     | 70.00th=[ 1960], 80.00th=[ 2192], 90.00th=[ 3920], 95.00th=[37120],
     | 99.00th=[68096], 99.50th=[95744], 99.90th=[201728], 99.95th=[203776],
     | 99.99th=[222208]
    bw (KB  /s): min=    4, max=1060864, per=100.00%, avg=367693.88, stdev=90931.84
    lat (usec) : 500=0.01%, 750=1.25%, 1000=4.55%
    lat (msec) : 2=69.36%, 4=14.95%, 10=2.12%, 20=0.26%, 50=5.12%
    lat (msec) : 100=1.95%, 250=0.44%, 500=0.01%, 750=0.01%
  cpu          : usr=1.65%, sys=3.52%, ctx=20100748, majf=0, minf=67499
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10300226/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=201809: Thu Jan 14 07:12:23 2016
  write: io=2474.8GB, bw=360405KB/s, iops=1407, runt=7200002msec
    slat (usec): min=24, max=289, avg=46.58, stdev=10.15
    clat (usec): min=420, max=419161, avg=5634.36, stdev=15573.33
     lat (usec): min=542, max=419208, avg=5681.13, stdev=15573.09
    clat percentiles (usec):
     |  1.00th=[  708],  5.00th=[  948], 10.00th=[ 1176], 20.00th=[ 1448],
     | 30.00th=[ 1576], 40.00th=[ 1704], 50.00th=[ 1816], 60.00th=[ 1832],
     | 70.00th=[ 1960], 80.00th=[ 2192], 90.00th=[ 3824], 95.00th=[39680],
     | 99.00th=[71168], 99.50th=[96768], 99.90th=[201728], 99.95th=[203776],
     | 99.99th=[211968]
    bw (KB  /s): min=    4, max=1060864, per=100.00%, avg=361757.91, stdev=89552.53
    lat (usec) : 500=0.01%, 750=1.24%, 1000=4.87%
    lat (msec) : 2=69.03%, 4=15.13%, 10=2.22%, 20=0.28%, 50=4.37%
    lat (msec) : 100=2.41%, 250=0.44%, 500=0.01%
  cpu          : usr=1.63%, sys=3.49%, ctx=19791038, majf=0, minf=66454
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10136376/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=201810: Thu Jan 14 07:12:23 2016
  write: io=906395MB, bw=128910KB/s, iops=503, runt=7200001msec
    slat (usec): min=28, max=2554, avg=73.55, stdev=12.80
    clat (usec): min=404, max=400424, avg=1909.36, stdev=8776.51
     lat (usec): min=486, max=400490, avg=1983.31, stdev=8775.91
    clat percentiles (usec):
     |  1.00th=[  502],  5.00th=[  524], 10.00th=[  532], 20.00th=[  540],
     | 30.00th=[  548], 40.00th=[  564], 50.00th=[  580], 60.00th=[  612],
     | 70.00th=[  724], 80.00th=[ 1512], 90.00th=[ 1896], 95.00th=[ 2096],
     | 99.00th=[47360], 99.50th=[66048], 99.90th=[109056], 99.95th=[144384],
     | 99.99th=[203776]
    bw (KB  /s): min=    4, max=300032, per=100.00%, avg=129479.14, stdev=36930.19
    lat (usec) : 500=0.74%, 750=69.85%, 1000=3.80%
    lat (msec) : 2=19.59%, 4=3.90%, 10=0.34%, 20=0.06%, 50=0.80%
    lat (msec) : 100=0.78%, 250=0.14%, 500=0.01%
  cpu          : usr=0.91%, sys=2.15%, ctx=7306751, majf=0, minf=9138
  IO depths    : 1=100.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3625581/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=201811: Thu Jan 14 07:12:23 2016
  write: io=894909MB, bw=127276KB/s, iops=497, runt=7200002msec
    slat (usec): min=28, max=2044, avg=73.95, stdev=12.90
    clat (usec): min=397, max=439874, avg=1934.43, stdev=8897.67
     lat (usec): min=462, max=439953, avg=2008.78, stdev=8897.09
    clat percentiles (usec):
     |  1.00th=[  502],  5.00th=[  524], 10.00th=[  532], 20.00th=[  540],
     | 30.00th=[  548], 40.00th=[  564], 50.00th=[  580], 60.00th=[  612],
     | 70.00th=[  716], 80.00th=[ 1448], 90.00th=[ 1864], 95.00th=[ 2096],
     | 99.00th=[48896], 99.50th=[67072], 99.90th=[109056], 99.95th=[144384],
     | 99.99th=[203776]
    bw (KB  /s): min=    4, max=305664, per=100.00%, avg=127837.36, stdev=36465.02
    lat (usec) : 500=0.81%, 750=70.15%, 1000=3.86%
    lat (msec) : 2=19.18%, 4=3.84%, 10=0.34%, 20=0.06%, 50=0.80%
    lat (msec) : 100=0.83%, 250=0.14%, 500=0.01%
  cpu          : usr=0.90%, sys=2.13%, ctx=7222138, majf=0, minf=9636
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3579634/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=2514.8GB, aggrb=366230KB/s, minb=366230KB/s, maxb=366230KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
  WRITE: io=2474.8GB, aggrb=360404KB/s, minb=360404KB/s, maxb=360404KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=906395MB, aggrb=128909KB/s, minb=128909KB/s, maxb=128909KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=894909MB, aggrb=127275KB/s, minb=127275KB/s, maxb=127275KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 07:12:25 AM EST
# 256kB read with SATA 1 jobs and 8 I/Os Outstanding
# 256kB read with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 2317 IOPS | 593 MBps | 3450 μsec Average Latency 
SATA-R5-1: 2324 IOPS | 595 MBps | 3439 μsec Average Latency 
NVMe-R5-0: 1968 IOPS | 503 MBps | 2028 μsec Average Latency 
NVMe-R5-1: 2021 IOPS | 517 MBps | 1977 μsec Average Latency 

Totals
IOPS: 8630
MBPS: 2208
Average Latency (μsec): 2723

SATA-R5-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=206861: Thu Jan 14 09:13:26 2016
  read : io=4073.4GB, bw=593220KB/s, iops=2317, runt=7200003msec
    slat (usec): min=18, max=1458, avg=43.55, stdev=19.75
    clat (usec): min=775, max=26941, avg=3406.82, stdev=840.49
     lat (usec): min=804, max=27006, avg=3450.60, stdev=839.98
    clat percentiles (usec):
     |  1.00th=[ 2256],  5.00th=[ 2672], 10.00th=[ 2832], 20.00th=[ 2960],
     | 30.00th=[ 3088], 40.00th=[ 3152], 50.00th=[ 3216], 60.00th=[ 3344],
     | 70.00th=[ 3440], 80.00th=[ 3632], 90.00th=[ 3984], 95.00th=[ 4768],
     | 99.00th=[ 7136], 99.50th=[ 8096], 99.90th=[10688], 99.95th=[11968],
     | 99.99th=[15040]
    bw (KB  /s): min=    4, max=724992, per=100.00%, avg=593507.48, stdev=12707.45
    lat (usec) : 1000=0.01%
    lat (msec) : 2=0.35%, 4=89.71%, 10=9.79%, 20=0.15%, 50=0.01%
  cpu          : usr=0.93%, sys=7.61%, ctx=33242283, majf=0, minf=67365
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=16684307/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=206862: Thu Jan 14 09:13:26 2016
  read : io=4086.8GB, bw=595171KB/s, iops=2324, runt=7200003msec
    slat (usec): min=16, max=2072, avg=41.51, stdev=18.61
    clat (usec): min=614, max=26303, avg=3397.42, stdev=867.31
     lat (usec): min=648, max=26380, avg=3439.18, stdev=866.88
    clat percentiles (usec):
     |  1.00th=[ 2160],  5.00th=[ 2608], 10.00th=[ 2736], 20.00th=[ 2928],
     | 30.00th=[ 3056], 40.00th=[ 3120], 50.00th=[ 3216], 60.00th=[ 3344],
     | 70.00th=[ 3472], 80.00th=[ 3632], 90.00th=[ 4048], 95.00th=[ 4832],
     | 99.00th=[ 7200], 99.50th=[ 8160], 99.90th=[10688], 99.95th=[11968],
     | 99.99th=[14912]
    bw (KB  /s): min=    4, max=755200, per=100.00%, avg=595472.94, stdev=14476.99
    lat (usec) : 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.51%, 4=89.03%, 10=10.31%, 20=0.15%, 50=0.01%
  cpu          : usr=0.93%, sys=7.45%, ctx=33351359, majf=0, minf=67886
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=16739198/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=206863: Thu Jan 14 09:13:26 2016
  read : io=3459.5GB, bw=503818KB/s, iops=1968, runt=7200002msec
    slat (usec): min=18, max=3071, avg=71.00, stdev=31.78
    clat (usec): min=154, max=23625, avg=1956.75, stdev=624.25
     lat (usec): min=361, max=23689, avg=2028.05, stdev=622.17
    clat percentiles (usec):
     |  1.00th=[  972],  5.00th=[ 1304], 10.00th=[ 1464], 20.00th=[ 1624],
     | 30.00th=[ 1720], 40.00th=[ 1800], 50.00th=[ 1880], 60.00th=[ 1976],
     | 70.00th=[ 2096], 80.00th=[ 2224], 90.00th=[ 2416], 95.00th=[ 2608],
     | 99.00th=[ 4768], 99.50th=[ 5856], 99.90th=[ 8256], 99.95th=[ 9536],
     | 99.99th=[12352]
    bw (KB  /s): min=    4, max=559104, per=100.00%, avg=504109.74, stdev=12454.43
    lat (usec) : 250=0.01%, 500=0.02%, 750=0.22%, 1000=0.92%
    lat (msec) : 2=61.05%, 4=36.44%, 10=1.31%, 20=0.04%, 50=0.01%
  cpu          : usr=1.48%, sys=8.62%, ctx=27647846, majf=0, minf=33980
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=14169893/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=206864: Thu Jan 14 09:13:26 2016
  read : io=3552.8GB, bw=517400KB/s, iops=2021, runt=7200002msec
    slat (usec): min=17, max=1467, avg=46.24, stdev=21.04
    clat (usec): min=208, max=24833, avg=1930.62, stdev=636.57
     lat (usec): min=358, max=24919, avg=1977.14, stdev=635.63
    clat percentiles (usec):
     |  1.00th=[  940],  5.00th=[ 1256], 10.00th=[ 1416], 20.00th=[ 1576],
     | 30.00th=[ 1688], 40.00th=[ 1768], 50.00th=[ 1848], 60.00th=[ 1960],
     | 70.00th=[ 2064], 80.00th=[ 2224], 90.00th=[ 2416], 95.00th=[ 2608],
     | 99.00th=[ 4768], 99.50th=[ 5920], 99.90th=[ 8256], 99.95th=[ 9536],
     | 99.99th=[12480]
    bw (KB  /s): min=    4, max=596480, per=100.00%, avg=517691.78, stdev=15352.28
    lat (usec) : 250=0.01%, 500=0.03%, 750=0.26%, 1000=1.11%
    lat (msec) : 2=63.19%, 4=34.07%, 10=1.30%, 20=0.04%, 50=0.01%
  cpu          : usr=0.90%, sys=6.97%, ctx=28963031, majf=0, minf=34102
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=14551884/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=4073.4GB, aggrb=593219KB/s, minb=593219KB/s, maxb=593219KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=4086.8GB, aggrb=595171KB/s, minb=595171KB/s, maxb=595171KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=3459.5GB, aggrb=503818KB/s, minb=503818KB/s, maxb=503818KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=3552.8GB, aggrb=517400KB/s, minb=517400KB/s, maxb=517400KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:13:28 AM EST
# 64kB write with SATA 1 jobs and 4 I/Os Outstanding
# 64kB write with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6183 IOPS | 395 MBps | 645 μsec Average Latency 
SATA-R5-1: 6438 IOPS | 412 MBps | 620 μsec Average Latency 
NVMe-R5-0: 2063 IOPS | 132 MBps | 1937 μsec Average Latency 
NVMe-R5-1: 1990 IOPS | 127 MBps | 2008 μsec Average Latency 

Totals
IOPS: 16674
MBPS: 1066
Average Latency (μsec): 1302

SATA-R5-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-01: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=211948: Thu Jan 14 11:14:29 2016
  write: io=2717.4GB, bw=395738KB/s, iops=6183, runt=7200001msec
    slat (usec): min=7, max=642, avg=23.08, stdev= 5.73
    clat (usec): min=130, max=399604, avg=622.46, stdev=3809.23
     lat (usec): min=195, max=399627, avg=645.73, stdev=3809.19
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  338], 10.00th=[  342], 20.00th=[  350],
     | 30.00th=[  358], 40.00th=[  382], 50.00th=[  438], 60.00th=[  458],
     | 70.00th=[  474], 80.00th=[  482], 90.00th=[  548], 95.00th=[  620],
     | 99.00th=[ 1352], 99.50th=[ 3664], 99.90th=[44288], 99.95th=[84480],
     | 99.99th=[191488]
    bw (KB  /s): min=    1, max=594688, per=100.00%, avg=397718.88, stdev=95110.90
    lat (usec) : 250=0.32%, 500=85.48%, 750=11.48%, 1000=1.23%
    lat (msec) : 2=0.81%, 4=0.19%, 10=0.09%, 20=0.05%, 50=0.26%
    lat (msec) : 100=0.05%, 250=0.04%, 500=0.01%
  cpu          : usr=2.53%, sys=6.36%, ctx=78297922, majf=0, minf=9449
  IO depths    : 1=0.1%, 2=0.1%, 4=101.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=44520489/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=211949: Thu Jan 14 11:14:29 2016
  write: io=2829.4GB, bw=412048KB/s, iops=6438, runt=7200001msec
    slat (usec): min=7, max=1504, avg=23.09, stdev= 5.66
    clat (usec): min=124, max=387865, avg=596.86, stdev=3240.55
     lat (usec): min=189, max=387888, avg=620.15, stdev=3240.51
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  338], 10.00th=[  346], 20.00th=[  350],
     | 30.00th=[  362], 40.00th=[  386], 50.00th=[  442], 60.00th=[  466],
     | 70.00th=[  474], 80.00th=[  482], 90.00th=[  556], 95.00th=[  620],
     | 99.00th=[ 1336], 99.50th=[ 2704], 99.90th=[40192], 99.95th=[70144],
     | 99.99th=[150528]
    bw (KB  /s): min=    1, max=580864, per=100.00%, avg=412975.15, stdev=70296.31
    lat (usec) : 250=0.30%, 500=85.22%, 750=11.69%, 1000=1.27%
    lat (msec) : 2=0.89%, 4=0.21%, 10=0.09%, 20=0.03%, 50=0.22%
    lat (msec) : 100=0.05%, 250=0.02%, 500=0.01%
  cpu          : usr=2.61%, sys=6.64%, ctx=81599743, majf=0, minf=9450
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=46355418/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=211950: Thu Jan 14 11:14:29 2016
  write: io=928412MB, bw=132041KB/s, iops=2063, runt=7200001msec
    slat (usec): min=7, max=2286, avg=22.35, stdev= 9.09
    clat (usec): min=140, max=475545, avg=1915.15, stdev=12744.73
     lat (usec): min=195, max=475566, avg=1937.69, stdev=12744.70
    clat percentiles (usec):
     |  1.00th=[  302],  5.00th=[  338], 10.00th=[  346], 20.00th=[  366],
     | 30.00th=[  406], 40.00th=[  450], 50.00th=[  470], 60.00th=[  482],
     | 70.00th=[  498], 80.00th=[  564], 90.00th=[  636], 95.00th=[  828],
     | 99.00th=[50944], 99.50th=[93696], 99.90th=[199680], 99.95th=[199680],
     | 99.99th=[264192]
    bw (KB  /s): min=    1, max=568192, per=100.00%, avg=133292.13, stdev=51183.89
    lat (usec) : 250=0.17%, 500=70.60%, 750=23.15%, 1000=2.43%
    lat (msec) : 2=1.33%, 4=0.30%, 10=0.14%, 20=0.05%, 50=0.81%
    lat (msec) : 100=0.57%, 250=0.44%, 500=0.01%
  cpu          : usr=0.88%, sys=2.16%, ctx=25917898, majf=0, minf=9381
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=14854592/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=211951: Thu Jan 14 11:14:29 2016
  write: io=895882MB, bw=127414KB/s, iops=1990, runt=7200001msec
    slat (usec): min=7, max=151, avg=22.24, stdev= 9.29
    clat (usec): min=137, max=420028, avg=1985.64, stdev=13199.13
     lat (usec): min=201, max=420064, avg=2008.07, stdev=13199.10
    clat percentiles (usec):
     |  1.00th=[  306],  5.00th=[  338], 10.00th=[  346], 20.00th=[  370],
     | 30.00th=[  418], 40.00th=[  454], 50.00th=[  474], 60.00th=[  482],
     | 70.00th=[  502], 80.00th=[  564], 90.00th=[  644], 95.00th=[  836],
     | 99.00th=[53504], 99.50th=[98816], 99.90th=[199680], 99.95th=[201728],
     | 99.99th=[276480]
    bw (KB  /s): min=    1, max=532992, per=100.00%, avg=128592.85, stdev=49327.35
    lat (usec) : 250=0.15%, 500=69.65%, 750=23.98%, 1000=2.47%
    lat (msec) : 2=1.39%, 4=0.31%, 10=0.14%, 20=0.05%, 50=0.77%
    lat (msec) : 100=0.60%, 250=0.47%, 500=0.02%
  cpu          : usr=0.82%, sys=2.12%, ctx=24996875, majf=0, minf=9434
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=14334108/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=2717.4GB, aggrb=395737KB/s, minb=395737KB/s, maxb=395737KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=2829.4GB, aggrb=412048KB/s, minb=412048KB/s, maxb=412048KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=928412MB, aggrb=132040KB/s, minb=132040KB/s, maxb=132040KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
  WRITE: io=895882MB, aggrb=127414KB/s, minb=127414KB/s, maxb=127414KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 11:14:31 AM EST
# 64kB read with SATA 1 jobs and 10 I/Os Outstanding
# 64kB read with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 7490 IOPS | 479 MBps | 1334 μsec Average Latency 
SATA-R5-1: 7401 IOPS | 473 MBps | 1350 μsec Average Latency 
NVMe-R5-0: 9727 IOPS | 622 MBps | 821 μsec Average Latency 
NVMe-R5-1: 9563 IOPS | 612 MBps | 835 μsec Average Latency 

Totals
IOPS: 34181
MBPS: 2186
Average Latency (μsec): 1085

SATA-R5-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
SATA-R5-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
NVMe-R5-01: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=216999: Thu Jan 14 13:15:33 2016
  read : io=3291.7GB, bw=479384KB/s, iops=7490, runt=7200002msec
    slat (usec): min=5, max=1297, avg=11.42, stdev= 4.72
    clat (usec): min=78, max=22174, avg=1322.72, stdev=459.18
     lat (usec): min=184, max=22187, avg=1334.27, stdev=459.04
    clat percentiles (usec):
     |  1.00th=[  628],  5.00th=[  852], 10.00th=[  964], 20.00th=[ 1064],
     | 30.00th=[ 1112], 40.00th=[ 1224], 50.00th=[ 1240], 60.00th=[ 1352],
     | 70.00th=[ 1416], 80.00th=[ 1512], 90.00th=[ 1720], 95.00th=[ 1880],
     | 99.00th=[ 2928], 99.50th=[ 3536], 99.90th=[ 6112], 99.95th=[ 8384],
     | 99.99th=[10944]
    bw (KB  /s): min=    1, max=569856, per=100.00%, avg=479494.52, stdev=12943.08
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.26%, 750=2.74%, 1000=13.78%
    lat (msec) : 2=79.79%, 4=3.11%, 10=0.29%, 20=0.02%, 50=0.01%
  cpu          : usr=1.21%, sys=6.33%, ctx=88105510, majf=0, minf=21790
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=53930691/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=217000: Thu Jan 14 13:15:33 2016
  read : io=3252.5GB, bw=473671KB/s, iops=7401, runt=7200001msec
    slat (usec): min=6, max=1570, avg=20.51, stdev=18.66
    clat (usec): min=81, max=20337, avg=1329.54, stdev=516.53
     lat (usec): min=187, max=20351, avg=1350.20, stdev=515.88
    clat percentiles (usec):
     |  1.00th=[  604],  5.00th=[  796], 10.00th=[  892], 20.00th=[ 1020],
     | 30.00th=[ 1112], 40.00th=[ 1192], 50.00th=[ 1256], 60.00th=[ 1336],
     | 70.00th=[ 1432], 80.00th=[ 1544], 90.00th=[ 1736], 95.00th=[ 1976],
     | 99.00th=[ 3376], 99.50th=[ 3920], 99.90th=[ 6368], 99.95th=[ 8384],
     | 99.99th=[11072]
    bw (KB  /s): min=    1, max=565632, per=100.00%, avg=473784.82, stdev=14602.42
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.39%, 750=3.39%, 1000=14.33%
    lat (msec) : 2=77.18%, 4=4.25%, 10=0.43%, 20=0.02%, 50=0.01%
  cpu          : usr=1.22%, sys=8.80%, ctx=84095052, majf=0, minf=21977
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=53288012/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=217001: Thu Jan 14 13:15:33 2016
  read : io=4274.2GB, bw=622589KB/s, iops=9727, runt=7200001msec
    slat (usec): min=6, max=1212, avg=11.22, stdev= 4.11
    clat (usec): min=91, max=17284, avg=810.27, stdev=338.09
     lat (usec): min=170, max=17296, avg=821.61, stdev=337.95
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  482], 10.00th=[  510], 20.00th=[  612],
     | 30.00th=[  628], 40.00th=[  724], 50.00th=[  740], 60.00th=[  844],
     | 70.00th=[  876], 80.00th=[  988], 90.00th=[ 1128], 95.00th=[ 1256],
     | 99.00th=[ 1656], 99.50th=[ 1944], 99.90th=[ 4384], 99.95th=[ 6432],
     | 99.99th=[10176]
    bw (KB  /s): min=    1, max=668416, per=100.00%, avg=622686.14, stdev=15145.00
    lat (usec) : 100=0.01%, 250=0.01%, 500=8.42%, 750=44.07%, 1000=30.09%
    lat (msec) : 2=16.97%, 4=0.33%, 10=0.10%, 20=0.01%
  cpu          : usr=1.42%, sys=8.11%, ctx=111788491, majf=0, minf=18290
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=70041260/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=217002: Thu Jan 14 13:15:33 2016
  read : io=4202.7GB, bw=612050KB/s, iops=9563, runt=7200001msec
    slat (usec): min=5, max=1323, avg=20.39, stdev=18.45
    clat (usec): min=1, max=22562, avg=815.06, stdev=355.76
     lat (usec): min=174, max=22599, avg=835.61, stdev=354.81
    clat percentiles (usec):
     |  1.00th=[  362],  5.00th=[  462], 10.00th=[  502], 20.00th=[  588],
     | 30.00th=[  636], 40.00th=[  700], 50.00th=[  748], 60.00th=[  828],
     | 70.00th=[  900], 80.00th=[ 1004], 90.00th=[ 1176], 95.00th=[ 1336],
     | 99.00th=[ 1704], 99.50th=[ 1976], 99.90th=[ 4384], 99.95th=[ 6432],
     | 99.99th=[10176]
    bw (KB  /s): min=    1, max=652544, per=100.00%, avg=612142.03, stdev=16175.73
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.06%
    lat (usec) : 500=9.78%, 750=40.32%, 1000=29.04%
    lat (msec) : 2=20.34%, 4=0.36%, 10=0.10%, 20=0.01%, 50=0.01%
  cpu          : usr=1.51%, sys=11.08%, ctx=105615890, majf=0, minf=17872
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=68855584/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3291.7GB, aggrb=479383KB/s, minb=479383KB/s, maxb=479383KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=3252.5GB, aggrb=473671KB/s, minb=473671KB/s, maxb=473671KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=4274.2GB, aggrb=622588KB/s, minb=622588KB/s, maxb=622588KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=4202.7GB, aggrb=612049KB/s, minb=612049KB/s, maxb=612049KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 01:15:34 PM EST
# 64kB randread with SATA 2 jobs and 2 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 3867 IOPS | 247 MBps | 1032 μsec Average Latency 
SATA-R5-1: 3841 IOPS | 245 MBps | 1039 μsec Average Latency 
NVMe-R5-0: 9176 IOPS | 587 MBps | 869 μsec Average Latency 
NVMe-R5-1: 7688 IOPS | 492 MBps | 1038 μsec Average Latency 

Totals
IOPS: 24572
MBPS: 1571
Average Latency (μsec): 994

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 12 processes

SATA-R5-01: (groupid=0, jobs=2): err= 0: pid=222095: Thu Jan 14 15:16:36 2016
  read : io=1699.9GB, bw=247550KB/s, iops=3867, runt=7200001msec
    slat (usec): min=6, max=1099, avg=25.14, stdev=22.57
    clat (usec): min=116, max=18371, avg=1006.65, stdev=341.47
     lat (usec): min=196, max=18382, avg=1032.00, stdev=340.24
    clat percentiles (usec):
     |  1.00th=[  668],  5.00th=[  740], 10.00th=[  788], 20.00th=[  852],
     | 30.00th=[  884], 40.00th=[  932], 50.00th=[  972], 60.00th=[ 1004],
     | 70.00th=[ 1048], 80.00th=[ 1112], 90.00th=[ 1208], 95.00th=[ 1304],
     | 99.00th=[ 1800], 99.50th=[ 2960], 99.90th=[ 5600], 99.95th=[ 6688],
     | 99.99th=[10944]
    bw (KB  /s): min=    1, max=131072, per=50.03%, avg=123849.99, stdev=3148.30
    lat (usec) : 250=0.01%, 500=0.03%, 750=5.85%, 1000=52.49%
    lat (msec) : 2=40.75%, 4=0.67%, 10=0.19%, 20=0.02%
  cpu          : usr=0.76%, sys=3.11%, ctx=54341474, majf=0, minf=11275
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=27849347/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=2): err= 0: pid=222097: Thu Jan 14 15:16:36 2016
  read : io=1688.2GB, bw=245855KB/s, iops=3841, runt=7200001msec
    slat (usec): min=6, max=1023, avg=29.02, stdev=24.18
    clat (usec): min=46, max=21441, avg=1009.79, stdev=343.25
     lat (usec): min=188, max=21504, avg=1039.04, stdev=341.72
    clat percentiles (usec):
     |  1.00th=[  604],  5.00th=[  716], 10.00th=[  772], 20.00th=[  844],
     | 30.00th=[  892], 40.00th=[  940], 50.00th=[  980], 60.00th=[ 1020],
     | 70.00th=[ 1080], 80.00th=[ 1128], 90.00th=[ 1224], 95.00th=[ 1320],
     | 99.00th=[ 1656], 99.50th=[ 2672], 99.90th=[ 5664], 99.95th=[ 6944],
     | 99.99th=[11456]
    bw (KB  /s): min=    1, max=132096, per=50.03%, avg=123003.15, stdev=3250.31
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.10%, 750=7.90%
    lat (usec) : 1000=46.57%
    lat (msec) : 2=44.73%, 4=0.49%, 10=0.18%, 20=0.02%, 50=0.01%
  cpu          : usr=0.72%, sys=3.07%, ctx=53612221, majf=0, minf=10933
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=27658675/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=222099: Thu Jan 14 15:16:36 2016
  read : io=4032.6GB, bw=587283KB/s, iops=9176, runt=7200001msec
    slat (usec): min=5, max=2831, avg=24.38, stdev=21.18
    clat (usec): min=1, max=17533, avg=845.37, stdev=359.21
     lat (usec): min=168, max=17621, avg=869.96, stdev=358.17
    clat percentiles (usec):
     |  1.00th=[  318],  5.00th=[  442], 10.00th=[  564], 20.00th=[  652],
     | 30.00th=[  716], 40.00th=[  748], 50.00th=[  804], 60.00th=[  852],
     | 70.00th=[  908], 80.00th=[ 1004], 90.00th=[ 1160], 95.00th=[ 1320],
     | 99.00th=[ 1768], 99.50th=[ 2064], 99.90th=[ 5280], 99.95th=[ 6560],
     | 99.99th=[10688]
    bw (KB  /s): min=    1, max=184064, per=25.01%, avg=146893.73, stdev=7801.42
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.25%, 500=6.94%, 750=32.65%, 1000=40.10%
    lat (msec) : 2=19.52%, 4=0.40%, 10=0.14%, 20=0.01%
  cpu          : usr=0.79%, sys=3.65%, ctx=128243592, majf=0, minf=21976
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=66069307/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=222103: Thu Jan 14 15:16:36 2016
  read : io=3378.8GB, bw=492061KB/s, iops=7688, runt=7200001msec
    slat (usec): min=6, max=1294, avg=29.29, stdev=24.25
    clat (usec): min=1, max=25642, avg=1008.84, stdev=647.46
     lat (usec): min=169, max=25658, avg=1038.37, stdev=646.71
    clat percentiles (usec):
     |  1.00th=[  314],  5.00th=[  466], 10.00th=[  572], 20.00th=[  660],
     | 30.00th=[  716], 40.00th=[  764], 50.00th=[  828], 60.00th=[  892],
     | 70.00th=[  988], 80.00th=[ 1176], 90.00th=[ 1688], 95.00th=[ 2256],
     | 99.00th=[ 3600], 99.50th=[ 4256], 99.90th=[ 6112], 99.95th=[ 7264],
     | 99.99th=[11456]
    bw (KB  /s): min=    1, max=170112, per=25.01%, avg=123088.11, stdev=12319.41
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.28%
    lat (usec) : 500=5.91%, 750=30.32%, 1000=34.19%
    lat (msec) : 2=22.43%, 4=6.21%, 10=0.63%, 20=0.02%, 50=0.01%
  cpu          : usr=0.70%, sys=3.13%, ctx=107422198, majf=0, minf=22288
  IO depths    : 1=0.1%, 2=101.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=55356905/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1699.9GB, aggrb=247549KB/s, minb=247549KB/s, maxb=247549KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1688.2GB, aggrb=245854KB/s, minb=245854KB/s, maxb=245854KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=4032.6GB, aggrb=587282KB/s, minb=587282KB/s, maxb=587282KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=3378.8GB, aggrb=492061KB/s, minb=492061KB/s, maxb=492061KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 03:16:37 PM EST
# 64kB randread with SATA 6 jobs and 4 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6402 IOPS | 409 MBps | 3745 μsec Average Latency 
SATA-R5-1: 7601 IOPS | 486 MBps | 3155 μsec Average Latency 
NVMe-R5-0: 9222 IOPS | 590 MBps | 3468 μsec Average Latency 
NVMe-R5-1: 7966 IOPS | 509 MBps | 4014 μsec Average Latency 

Totals
IOPS: 31191
MBPS: 1994
Average Latency (μsec): 3595

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=227189: Thu Jan 14 17:17:39 2016
  read : io=2813.9GB, bw=409788KB/s, iops=6402, runt=7200004msec
    slat (usec): min=5, max=2662, avg=32.58, stdev=32.78
    clat (usec): min=89, max=24974, avg=3713.15, stdev=1371.75
     lat (usec): min=385, max=24991, avg=3745.96, stdev=1370.65
    clat percentiles (usec):
     |  1.00th=[ 2224],  5.00th=[ 2640], 10.00th=[ 2832], 20.00th=[ 2992],
     | 30.00th=[ 3088], 40.00th=[ 3184], 50.00th=[ 3280], 60.00th=[ 3376],
     | 70.00th=[ 3536], 80.00th=[ 3856], 90.00th=[ 5600], 95.00th=[ 6880],
     | 99.00th=[ 9152], 99.50th=[10176], 99.90th=[12480], 99.95th=[13376],
     | 99.99th=[15296]
    bw (KB  /s): min=    1, max=82816, per=16.68%, avg=68348.81, stdev=1555.92
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.45%, 4=81.46%, 10=17.53%, 20=0.56%, 50=0.01%
  cpu          : usr=0.42%, sys=1.75%, ctx=90237993, majf=0, minf=60815
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=46101167/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=227195: Thu Jan 14 17:17:39 2016
  read : io=3340.7GB, bw=486517KB/s, iops=7601, runt=7200003msec
    slat (usec): min=5, max=3374, avg=29.42, stdev=30.41
    clat (usec): min=0, max=25445, avg=3125.32, stdev=1475.63
     lat (usec): min=233, max=25459, avg=3155.01, stdev=1474.72
    clat percentiles (usec):
     |  1.00th=[ 1464],  5.00th=[ 1832], 10.00th=[ 2024], 20.00th=[ 2256],
     | 30.00th=[ 2416], 40.00th=[ 2576], 50.00th=[ 2704], 60.00th=[ 2864],
     | 70.00th=[ 3056], 80.00th=[ 3408], 90.00th=[ 5088], 95.00th=[ 6560],
     | 99.00th=[ 8896], 99.50th=[ 9920], 99.90th=[12224], 99.95th=[12992],
     | 99.99th=[14912]
    bw (KB  /s): min=    1, max=106878, per=16.68%, avg=81144.99, stdev=5288.81
    lat (usec) : 2=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
    lat (usec) : 1000=0.05%
    lat (msec) : 2=9.04%, 4=77.38%, 10=13.05%, 20=0.47%, 50=0.01%
  cpu          : usr=0.47%, sys=2.07%, ctx=106391070, majf=0, minf=57255
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=54733215/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=227201: Thu Jan 14 17:17:39 2016
  read : io=4052.8GB, bw=590224KB/s, iops=9222, runt=7200002msec
    slat (usec): min=5, max=3213, avg=23.27, stdev=25.31
    clat (usec): min=32, max=25831, avg=3444.70, stdev=1358.97
     lat (usec): min=209, max=25851, avg=3468.15, stdev=1358.21
    clat percentiles (usec):
     |  1.00th=[ 1688],  5.00th=[ 2256], 10.00th=[ 2480], 20.00th=[ 2704],
     | 30.00th=[ 2832], 40.00th=[ 2960], 50.00th=[ 3024], 60.00th=[ 3152],
     | 70.00th=[ 3344], 80.00th=[ 3760], 90.00th=[ 5216], 95.00th=[ 6496],
     | 99.00th=[ 8768], 99.50th=[ 9792], 99.90th=[12224], 99.95th=[13120],
     | 99.99th=[15168]
    bw (KB  /s): min=    1, max=178560, per=25.01%, avg=147637.61, stdev=5045.50
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.02%
    lat (usec) : 1000=0.06%
    lat (msec) : 2=2.44%, 4=80.53%, 10=16.50%, 20=0.44%, 50=0.01%
  cpu          : usr=0.69%, sys=3.23%, ctx=125902047, majf=0, minf=74601
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=66400169/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=227205: Thu Jan 14 17:17:39 2016
  read : io=3501.4GB, bw=509875KB/s, iops=7966, runt=7200004msec
    slat (usec): min=5, max=2786, avg=28.57, stdev=29.41
    clat (usec): min=0, max=73137, avg=3985.93, stdev=3322.19
     lat (usec): min=205, max=73158, avg=4014.78, stdev=3321.95
    clat percentiles (usec):
     |  1.00th=[  884],  5.00th=[ 1368], 10.00th=[ 1624], 20.00th=[ 1912],
     | 30.00th=[ 2128], 40.00th=[ 2288], 50.00th=[ 2512], 60.00th=[ 2800],
     | 70.00th=[ 4128], 80.00th=[ 6304], 90.00th=[ 8384], 95.00th=[10304],
     | 99.00th=[17024], 99.50th=[19072], 99.90th=[24448], 99.95th=[27008],
     | 99.99th=[33536]
    bw (KB  /s): min=    1, max=179200, per=25.02%, avg=127551.52, stdev=11026.36
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.11%, 750=0.44%, 1000=1.07%
    lat (msec) : 2=22.22%, 4=45.81%, 10=24.74%, 20=5.22%, 50=0.38%
    lat (msec) : 100=0.01%
  cpu          : usr=0.64%, sys=2.92%, ctx=108545993, majf=0, minf=74852
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=101.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=57360936/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=2813.9GB, aggrb=409788KB/s, minb=409788KB/s, maxb=409788KB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=3340.7GB, aggrb=486517KB/s, minb=486517KB/s, maxb=486517KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=4052.8GB, aggrb=590223KB/s, minb=590223KB/s, maxb=590223KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=3501.4GB, aggrb=509874KB/s, minb=509874KB/s, maxb=509874KB/s, mint=7200004msec, maxt=7200004msec



# 
# Thu 14 Jan 2016 05:17:41 PM EST
# 32kB randread with SATA 5 jobs and 2 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 10048 IOPS | 321 MBps | 993 μsec Average Latency 
SATA-R5-1: 10278 IOPS | 328 MBps | 971 μsec Average Latency 
NVMe-R5-0: 14617 IOPS | 467 MBps | 819 μsec Average Latency 
NVMe-R5-1: 13623 IOPS | 435 MBps | 879 μsec Average Latency 

Totals
IOPS: 48566
MBPS: 1551
Average Latency (μsec): 915

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=232312: Thu Jan 14 19:18:42 2016
  read : io=2207.1GB, bw=321558KB/s, iops=10048, runt=7200001msec
    slat (usec): min=3, max=1199, avg=20.61, stdev=21.83
    clat (usec): min=16, max=22966, avg=972.55, stdev=533.91
     lat (usec): min=158, max=22977, avg=993.35, stdev=533.15
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  684], 10.00th=[  724], 20.00th=[  772],
     | 30.00th=[  828], 40.00th=[  860], 50.00th=[  892], 60.00th=[  940],
     | 70.00th=[  988], 80.00th=[ 1048], 90.00th=[ 1160], 95.00th=[ 1304],
     | 99.00th=[ 2960], 99.50th=[ 4832], 99.90th=[ 8384], 99.95th=[10304],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=75200, per=20.01%, avg=64349.77, stdev=2462.94
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.15%
    lat (usec) : 750=15.07%, 1000=56.69%
    lat (msec) : 2=26.69%, 4=0.63%, 10=0.72%, 20=0.06%, 50=0.01%
  cpu          : usr=0.67%, sys=2.51%, ctx=139508592, majf=0, minf=17606
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=72350457/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=232317: Thu Jan 14 19:18:42 2016
  read : io=2258.6GB, bw=328925KB/s, iops=10278, runt=7200001msec
    slat (usec): min=3, max=1343, avg=19.98, stdev=21.15
    clat (usec): min=1, max=23478, avg=950.95, stdev=541.61
     lat (usec): min=176, max=23485, avg=971.11, stdev=541.00
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  636], 10.00th=[  700], 20.00th=[  748],
     | 30.00th=[  796], 40.00th=[  844], 50.00th=[  876], 60.00th=[  916],
     | 70.00th=[  980], 80.00th=[ 1032], 90.00th=[ 1144], 95.00th=[ 1288],
     | 99.00th=[ 3024], 99.50th=[ 4832], 99.90th=[ 8512], 99.95th=[10432],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=75968, per=20.01%, avg=65822.71, stdev=2710.70
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.29%, 750=20.68%, 1000=54.00%
    lat (msec) : 2=23.62%, 4=0.64%, 10=0.71%, 20=0.06%, 50=0.01%
  cpu          : usr=0.67%, sys=2.54%, ctx=143030437, majf=0, minf=17361
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=74008035/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=232322: Thu Jan 14 19:18:42 2016
  read : io=3211.9GB, bw=467762KB/s, iops=14617, runt=7200001msec
    slat (usec): min=3, max=2290, avg=16.35, stdev=16.51
    clat (usec): min=1, max=22283, avg=803.25, stdev=511.12
     lat (usec): min=145, max=22296, avg=819.75, stdev=510.46
    clat percentiles (usec):
     |  1.00th=[  366],  5.00th=[  486], 10.00th=[  548], 20.00th=[  612],
     | 30.00th=[  652], 40.00th=[  700], 50.00th=[  732], 60.00th=[  764],
     | 70.00th=[  828], 80.00th=[  892], 90.00th=[ 1032], 95.00th=[ 1192],
     | 99.00th=[ 2160], 99.50th=[ 4448], 99.90th=[ 7904], 99.95th=[ 9792],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=183744, per=33.34%, avg=155965.53, stdev=8482.24
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.07%
    lat (usec) : 500=6.08%, 750=48.83%, 1000=33.35%
    lat (msec) : 2=10.59%, 4=0.46%, 10=0.58%, 20=0.05%, 50=0.01%
  cpu          : usr=0.92%, sys=4.60%, ctx=186587564, majf=0, minf=18106
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=105246353/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=232325: Thu Jan 14 19:18:42 2016
  read : io=2993.6GB, bw=435968KB/s, iops=13623, runt=7200001msec
    slat (usec): min=3, max=1546, avg=15.74, stdev=15.43
    clat (usec): min=0, max=24340, avg=863.64, stdev=614.53
     lat (usec): min=125, max=24348, avg=879.53, stdev=614.11
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  486], 10.00th=[  556], 20.00th=[  612],
     | 30.00th=[  652], 40.00th=[  708], 50.00th=[  740], 60.00th=[  780],
     | 70.00th=[  852], 80.00th=[  956], 90.00th=[ 1160], 95.00th=[ 1496],
     | 99.00th=[ 3728], 99.50th=[ 4832], 99.90th=[ 8384], 99.95th=[10304],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=186944, per=33.34%, avg=145369.77, stdev=14200.01
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.07%, 500=6.06%, 750=46.42%, 1000=31.00%
    lat (msec) : 2=13.79%, 4=1.81%, 10=0.81%, 20=0.06%, 50=0.01%
  cpu          : usr=0.99%, sys=4.35%, ctx=175990576, majf=0, minf=17673
  IO depths    : 1=0.1%, 2=0.1%, 4=101.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=98092713/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=2207.1GB, aggrb=321557KB/s, minb=321557KB/s, maxb=321557KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=2258.6GB, aggrb=328924KB/s, minb=328924KB/s, maxb=328924KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=3211.9GB, aggrb=467761KB/s, minb=467761KB/s, maxb=467761KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=2993.6GB, aggrb=435967KB/s, minb=435967KB/s, maxb=435967KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 07:18:44 PM EST
# 32kB randread with SATA 4 jobs and 16 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 18450 IOPS | 590 MBps | 3467 μsec Average Latency 
SATA-R5-1: 18610 IOPS | 595 MBps | 3437 μsec Average Latency 
NVMe-R5-0: 10121 IOPS | 323 MBps | 2369 μsec Average Latency 
NVMe-R5-1: 10138 IOPS | 324 MBps | 2365 μsec Average Latency 

Totals
IOPS: 57319
MBPS: 1832
Average Latency (μsec): 2909

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 14 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=237434: Thu Jan 14 21:19:46 2016
  read : io=4054.1GB, bw=590421KB/s, iops=18450, runt=7200002msec
    slat (usec): min=3, max=4031, avg=15.28, stdev=18.15
    clat (usec): min=1, max=129737, avg=3451.99, stdev=1511.45
     lat (usec): min=267, max=129751, avg=3467.42, stdev=1511.36
    clat percentiles (usec):
     |  1.00th=[ 1752],  5.00th=[ 2096], 10.00th=[ 2256], 20.00th=[ 2480],
     | 30.00th=[ 2672], 40.00th=[ 2864], 50.00th=[ 3056], 60.00th=[ 3248],
     | 70.00th=[ 3536], 80.00th=[ 3920], 90.00th=[ 5024], 95.00th=[ 7008],
     | 99.00th=[ 9280], 99.50th=[10432], 99.90th=[12736], 99.95th=[13632],
     | 99.99th=[15680]
    bw (KB  /s): min=    0, max=213888, per=25.01%, avg=147658.43, stdev=5053.96
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.01%, 1000=0.02%
    lat (msec) : 2=3.18%, 4=78.38%, 10=17.78%, 20=0.64%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.02%, sys=4.30%, ctx=232979185, majf=0, minf=74986
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=132844633/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=237438: Thu Jan 14 21:19:46 2016
  read : io=4089.2GB, bw=595532KB/s, iops=18610, runt=7200003msec
    slat (usec): min=3, max=2174, avg=18.02, stdev=21.31
    clat (usec): min=63, max=105390, avg=3419.53, stdev=1511.54
     lat (usec): min=222, max=105488, avg=3437.71, stdev=1511.20
    clat percentiles (usec):
     |  1.00th=[ 1720],  5.00th=[ 2064], 10.00th=[ 2224], 20.00th=[ 2448],
     | 30.00th=[ 2640], 40.00th=[ 2832], 50.00th=[ 3024], 60.00th=[ 3248],
     | 70.00th=[ 3504], 80.00th=[ 3888], 90.00th=[ 4960], 95.00th=[ 7008],
     | 99.00th=[ 9280], 99.50th=[10432], 99.90th=[12736], 99.95th=[13632],
     | 99.99th=[15680]
    bw (KB  /s): min=    0, max=215040, per=25.01%, avg=148939.11, stdev=5144.89
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.02%
    lat (msec) : 2=3.76%, 4=78.35%, 10=17.22%, 20=0.64%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.91%, sys=4.51%, ctx=229520946, majf=0, minf=75339
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=133994699/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=237442: Thu Jan 14 21:19:46 2016
  read : io=2223.9GB, bw=323875KB/s, iops=10121, runt=7200002msec
    slat (usec): min=3, max=4078, avg=15.94, stdev=19.35
    clat (usec): min=1, max=173686, avg=2353.84, stdev=1349.31
     lat (usec): min=187, max=173709, avg=2369.95, stdev=1349.44
    clat percentiles (usec):
     |  1.00th=[ 1048],  5.00th=[ 1368], 10.00th=[ 1496], 20.00th=[ 1624],
     | 30.00th=[ 1736], 40.00th=[ 1864], 50.00th=[ 1976], 60.00th=[ 2096],
     | 70.00th=[ 2320], 80.00th=[ 2640], 90.00th=[ 3312], 95.00th=[ 5536],
     | 99.00th=[ 8160], 99.50th=[ 9152], 99.90th=[11456], 99.95th=[12224],
     | 99.99th=[14272]
    bw (KB  /s): min=    0, max=125952, per=33.35%, avg=108009.37, stdev=4429.29
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.17%, 1000=0.64%
    lat (msec) : 2=52.62%, 4=39.82%, 10=6.46%, 20=0.28%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.81%, sys=3.32%, ctx=131900393, majf=0, minf=29467
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=72871847/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=237445: Thu Jan 14 21:19:46 2016
  read : io=2227.8GB, bw=324437KB/s, iops=10138, runt=7200002msec
    slat (usec): min=3, max=2553, avg=19.13, stdev=23.61
    clat (usec): min=30, max=153857, avg=2346.52, stdev=1353.73
     lat (usec): min=182, max=153870, avg=2365.83, stdev=1353.68
    clat percentiles (usec):
     |  1.00th=[ 1048],  5.00th=[ 1336], 10.00th=[ 1464], 20.00th=[ 1608],
     | 30.00th=[ 1736], 40.00th=[ 1848], 50.00th=[ 1960], 60.00th=[ 2096],
     | 70.00th=[ 2320], 80.00th=[ 2640], 90.00th=[ 3312], 95.00th=[ 5536],
     | 99.00th=[ 8160], 99.50th=[ 9152], 99.90th=[11456], 99.95th=[12352],
     | 99.99th=[14272]
    bw (KB  /s): min=    0, max=126080, per=33.35%, avg=108197.74, stdev=4659.32
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.15%
    lat (usec) : 1000=0.62%
    lat (msec) : 2=52.49%, 4=39.98%, 10=6.46%, 20=0.28%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.75%, sys=3.39%, ctx=129560391, majf=0, minf=30576
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=72998219/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=4054.1GB, aggrb=590420KB/s, minb=590420KB/s, maxb=590420KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=4089.2GB, aggrb=595532KB/s, minb=595532KB/s, maxb=595532KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=2223.9GB, aggrb=323874KB/s, minb=323874KB/s, maxb=323874KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=2227.8GB, aggrb=324436KB/s, minb=324436KB/s, maxb=324436KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:19:47 PM EST
# 16kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 16kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 28211 IOPS | 451 MBps | 2834 μsec Average Latency 
SATA-R5-1: 28197 IOPS | 451 MBps | 2836 μsec Average Latency 
NVMe-R5-0: 6419 IOPS | 102 MBps | 2490 μsec Average Latency 
NVMe-R5-1: 6637 IOPS | 106 MBps | 2408 μsec Average Latency 

Totals
IOPS: 69464
MBPS: 1110
Average Latency (μsec): 2642

SATA-R5-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=242546: Thu Jan 14 23:20:49 2016
  read : io=3099.5GB, bw=451390KB/s, iops=28211, runt=7200002msec
    slat (usec): min=2, max=3073, avg=14.05, stdev=12.54
    clat (usec): min=101, max=142703, avg=2820.36, stdev=581.99
     lat (usec): min=260, max=142711, avg=2834.55, stdev=581.50
    clat percentiles (usec):
     |  1.00th=[ 1928],  5.00th=[ 2192], 10.00th=[ 2288], 20.00th=[ 2448],
     | 30.00th=[ 2576], 40.00th=[ 2640], 50.00th=[ 2736], 60.00th=[ 2864],
     | 70.00th=[ 2960], 80.00th=[ 3120], 90.00th=[ 3344], 95.00th=[ 3600],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8160], 99.95th=[ 9152],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=203616, per=20.01%, avg=90301.24, stdev=2122.35
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=1.64%, 4=96.22%, 10=2.09%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.02%, sys=4.76%, ctx=346635589, majf=0, minf=50253
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=203125345/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=242551: Thu Jan 14 23:20:49 2016
  read : io=3097.9GB, bw=451156KB/s, iops=28197, runt=7200002msec
    slat (usec): min=2, max=3239, avg=12.50, stdev=10.96
    clat (usec): min=120, max=187181, avg=2823.42, stdev=581.62
     lat (usec): min=227, max=187189, avg=2836.04, stdev=581.27
    clat percentiles (usec):
     |  1.00th=[ 1928],  5.00th=[ 2192], 10.00th=[ 2320], 20.00th=[ 2480],
     | 30.00th=[ 2576], 40.00th=[ 2672], 50.00th=[ 2768], 60.00th=[ 2864],
     | 70.00th=[ 2960], 80.00th=[ 3120], 90.00th=[ 3344], 95.00th=[ 3600],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8256], 99.95th=[ 9152],
     | 99.99th=[12096]
    bw (KB  /s): min=    0, max=197696, per=20.01%, avg=90254.62, stdev=2117.36
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=1.64%, 4=96.30%, 10=2.01%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.12%, sys=4.32%, ctx=347954127, majf=0, minf=51105
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=203020132/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=242556: Thu Jan 14 23:20:49 2016
  read : io=722180MB, bw=102710KB/s, iops=6419, runt=7200001msec
    slat (usec): min=2, max=3105, avg=19.45, stdev=20.47
    clat (usec): min=15, max=99361, avg=2470.79, stdev=567.06
     lat (usec): min=204, max=99378, avg=2490.44, stdev=566.63
    clat percentiles (usec):
     |  1.00th=[ 1592],  5.00th=[ 1848], 10.00th=[ 1960], 20.00th=[ 2096],
     | 30.00th=[ 2224], 40.00th=[ 2320], 50.00th=[ 2416], 60.00th=[ 2512],
     | 70.00th=[ 2608], 80.00th=[ 2768], 90.00th=[ 2992], 95.00th=[ 3216],
     | 99.00th=[ 4192], 99.50th=[ 5152], 99.90th=[ 7776], 99.95th=[ 8640],
     | 99.99th=[11328]
    bw (KB  /s): min=    0, max=51168, per=25.02%, avg=25693.76, stdev=718.61
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.02%, 1000=0.04%
    lat (msec) : 2=12.31%, 4=86.41%, 10=1.20%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.56%, sys=1.76%, ctx=87134911, majf=0, minf=14472
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=46219517/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=242560: Thu Jan 14 23:20:49 2016
  read : io=746736MB, bw=106202KB/s, iops=6637, runt=7200001msec
    slat (usec): min=2, max=2367, avg=17.23, stdev=17.97
    clat (usec): min=27, max=95522, avg=2391.21, stdev=568.95
     lat (usec): min=217, max=95533, avg=2408.62, stdev=568.69
    clat percentiles (usec):
     |  1.00th=[ 1496],  5.00th=[ 1752], 10.00th=[ 1864], 20.00th=[ 2024],
     | 30.00th=[ 2128], 40.00th=[ 2224], 50.00th=[ 2352], 60.00th=[ 2448],
     | 70.00th=[ 2544], 80.00th=[ 2704], 90.00th=[ 2928], 95.00th=[ 3152],
     | 99.00th=[ 4080], 99.50th=[ 5024], 99.90th=[ 7712], 99.95th=[ 8640],
     | 99.99th=[11200]
    bw (KB  /s): min=    0, max=51360, per=25.02%, avg=26567.71, stdev=850.63
    lat (usec) : 50=0.01%, 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.05%
    lat (msec) : 2=18.38%, 4=80.47%, 10=1.07%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.59%, sys=1.84%, ctx=90520227, majf=0, minf=14818
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47791086/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=3099.5GB, aggrb=451389KB/s, minb=451389KB/s, maxb=451389KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=3097.9GB, aggrb=451155KB/s, minb=451155KB/s, maxb=451155KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=722180MB, aggrb=102710KB/s, minb=102710KB/s, maxb=102710KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=746736MB, aggrb=106202KB/s, minb=106202KB/s, maxb=106202KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 11:20:51 PM EST
# 8kB randread with SATA 6 jobs and 2 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 19751 IOPS | 158 MBps | 606 μsec Average Latency 
SATA-R5-1: 19886 IOPS | 159 MBps | 602 μsec Average Latency 
NVMe-R5-0: 14560 IOPS | 116 MBps | 548 μsec Average Latency 
NVMe-R5-1: 14612 IOPS | 116 MBps | 546 μsec Average Latency 

Totals
IOPS: 68809
MBPS: 549
Average Latency (μsec): 575

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=2639: Fri Jan 15 01:21:53 2016
  read : io=1084.2GB, bw=158013KB/s, iops=19751, runt=7200001msec
    slat (usec): min=2, max=1177, avg=11.85, stdev= 8.26
    clat (usec): min=15, max=26879, avg=594.15, stdev=196.79
     lat (usec): min=164, max=26890, avg=606.15, stdev=196.62
    clat percentiles (usec):
     |  1.00th=[  418],  5.00th=[  466], 10.00th=[  478], 20.00th=[  490],
     | 30.00th=[  506], 40.00th=[  540], 50.00th=[  588], 60.00th=[  604],
     | 70.00th=[  612], 80.00th=[  628], 90.00th=[  716], 95.00th=[  780],
     | 99.00th=[ 1176], 99.50th=[ 1400], 99.90th=[ 3088], 99.95th=[ 3504],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=30400, per=16.67%, avg=26345.21, stdev=521.73
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=26.83%
    lat (usec) : 750=66.98%, 1000=3.98%
    lat (msec) : 2=1.90%, 4=0.28%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=0.75%, sys=2.58%, ctx=266618187, majf=0, minf=13284
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=142211813/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=2645: Fri Jan 15 01:21:53 2016
  read : io=1092.4GB, bw=159090KB/s, iops=19886, runt=7200001msec
    slat (usec): min=2, max=1440, avg= 9.30, stdev= 6.64
    clat (usec): min=56, max=40567, avg=592.69, stdev=187.81
     lat (usec): min=129, max=40579, avg=602.13, stdev=187.66
    clat percentiles (usec):
     |  1.00th=[  418],  5.00th=[  470], 10.00th=[  482], 20.00th=[  490],
     | 30.00th=[  502], 40.00th=[  532], 50.00th=[  596], 60.00th=[  604],
     | 70.00th=[  620], 80.00th=[  628], 90.00th=[  716], 95.00th=[  764],
     | 99.00th=[ 1144], 99.50th=[ 1336], 99.90th=[ 2992], 99.95th=[ 3504],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=31536, per=16.67%, avg=26524.39, stdev=398.85
    lat (usec) : 100=0.01%, 250=0.01%, 500=28.28%, 750=65.82%, 1000=3.84%
    lat (msec) : 2=1.81%, 4=0.22%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=0.86%, sys=2.41%, ctx=272633959, majf=0, minf=11879
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=143181354/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=2651: Fri Jan 15 01:21:53 2016
  read : io=819015MB, bw=116482KB/s, iops=14560, runt=7200001msec
    slat (usec): min=2, max=7343, avg=11.58, stdev= 7.74
    clat (usec): min=10, max=16979, avg=536.38, stdev=174.91
     lat (usec): min=106, max=16986, avg=548.11, stdev=174.64
    clat percentiles (usec):
     |  1.00th=[  334],  5.00th=[  374], 10.00th=[  426], 20.00th=[  470],
     | 30.00th=[  482], 40.00th=[  490], 50.00th=[  498], 60.00th=[  516],
     | 70.00th=[  572], 80.00th=[  604], 90.00th=[  636], 95.00th=[  740],
     | 99.00th=[ 1064], 99.50th=[ 1208], 99.90th=[ 2736], 99.95th=[ 3344],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=34160, per=25.01%, avg=29129.88, stdev=584.54
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.11%, 500=51.34%
    lat (usec) : 750=44.14%, 1000=3.03%
    lat (msec) : 2=1.20%, 4=0.15%, 10=0.03%, 20=0.01%
  cpu          : usr=0.78%, sys=2.78%, ctx=196530615, majf=0, minf=8865
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=104833905/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=2655: Fri Jan 15 01:21:53 2016
  read : io=821971MB, bw=116902KB/s, iops=14612, runt=7200001msec
    slat (usec): min=2, max=494, avg= 9.12, stdev= 6.37
    clat (usec): min=54, max=53089, avg=536.86, stdev=173.64
     lat (usec): min=107, max=53096, avg=546.12, stdev=173.42
    clat percentiles (usec):
     |  1.00th=[  338],  5.00th=[  370], 10.00th=[  426], 20.00th=[  474],
     | 30.00th=[  482], 40.00th=[  490], 50.00th=[  498], 60.00th=[  510],
     | 70.00th=[  580], 80.00th=[  612], 90.00th=[  636], 95.00th=[  740],
     | 99.00th=[ 1064], 99.50th=[ 1208], 99.90th=[ 2704], 99.95th=[ 3344],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=34480, per=25.01%, avg=29234.73, stdev=488.91
    lat (usec) : 100=0.01%, 250=0.14%, 500=52.64%, 750=42.91%, 1000=2.99%
    lat (msec) : 2=1.16%, 4=0.14%, 10=0.03%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.96%, sys=2.54%, ctx=199440117, majf=0, minf=7863
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=105212222/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1084.2GB, aggrb=158013KB/s, minb=158013KB/s, maxb=158013KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1092.4GB, aggrb=159090KB/s, minb=159090KB/s, maxb=159090KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=819015MB, aggrb=116482KB/s, minb=116482KB/s, maxb=116482KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=821971MB, aggrb=116902KB/s, minb=116902KB/s, maxb=116902KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 01:21:55 AM EST
# 8kB randread with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 23921 IOPS | 191 MBps | 1336 μsec Average Latency 
SATA-R5-1: 24157 IOPS | 193 MBps | 1323 μsec Average Latency 
NVMe-R5-0: 12891 IOPS | 103 MBps | 1239 μsec Average Latency 
NVMe-R5-1: 12859 IOPS | 102 MBps | 1242 μsec Average Latency 

Totals
IOPS: 73828
MBPS: 589
Average Latency (μsec): 1285

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=8624: Fri Jan 15 03:22:56 2016
  read : io=1314.3GB, bw=191368KB/s, iops=23921, runt=7200001msec
    slat (usec): min=2, max=1843, avg=10.73, stdev= 6.62
    clat (usec): min=93, max=132192, avg=1325.80, stdev=415.06
     lat (usec): min=225, max=132201, avg=1336.67, stdev=414.78
    clat percentiles (usec):
     |  1.00th=[  764],  5.00th=[  956], 10.00th=[ 1004], 20.00th=[ 1096],
     | 30.00th=[ 1144], 40.00th=[ 1224], 50.00th=[ 1240], 60.00th=[ 1320],
     | 70.00th=[ 1368], 80.00th=[ 1480], 90.00th=[ 1672], 95.00th=[ 1880],
     | 99.00th=[ 2992], 99.50th=[ 3760], 99.90th=[ 5472], 99.95th=[ 6816],
     | 99.99th=[ 8384]
    bw (KB  /s): min=    0, max=99728, per=25.01%, avg=47851.92, stdev=1127.23
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.03%, 750=0.84%, 1000=8.87%
    lat (msec) : 2=86.76%, 4=3.12%, 10=0.38%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.08%, sys=3.95%, ctx=298317445, majf=0, minf=16172
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172231470/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=8628: Fri Jan 15 03:22:56 2016
  read : io=1327.3GB, bw=193263KB/s, iops=24157, runt=7200001msec
    slat (usec): min=2, max=1445, avg=10.77, stdev= 6.58
    clat (usec): min=56, max=85524, avg=1312.66, stdev=406.68
     lat (usec): min=222, max=85532, avg=1323.57, stdev=406.44
    clat percentiles (usec):
     |  1.00th=[  756],  5.00th=[  948], 10.00th=[  996], 20.00th=[ 1096],
     | 30.00th=[ 1128], 40.00th=[ 1208], 50.00th=[ 1240], 60.00th=[ 1288],
     | 70.00th=[ 1368], 80.00th=[ 1464], 90.00th=[ 1640], 95.00th=[ 1864],
     | 99.00th=[ 2896], 99.50th=[ 3728], 99.90th=[ 5344], 99.95th=[ 6752],
     | 99.99th=[ 8384]
    bw (KB  /s): min=    0, max=94960, per=25.01%, avg=48325.69, stdev=1081.37
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.03%, 750=0.89%, 1000=9.64%
    lat (msec) : 2=86.17%, 4=2.91%, 10=0.36%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.10%, sys=4.01%, ctx=301329495, majf=0, minf=14971
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=173936258/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=8632: Fri Jan 15 03:22:56 2016
  read : io=725123MB, bw=103129KB/s, iops=12891, runt=7200001msec
    slat (usec): min=1, max=1357, avg=10.05, stdev= 6.64
    clat (usec): min=31, max=81593, avg=1229.74, stdev=379.79
     lat (usec): min=110, max=81601, avg=1239.93, stdev=379.69
    clat percentiles (usec):
     |  1.00th=[  700],  5.00th=[  860], 10.00th=[  948], 20.00th=[ 1004],
     | 30.00th=[ 1096], 40.00th=[ 1112], 50.00th=[ 1176], 60.00th=[ 1240],
     | 70.00th=[ 1272], 80.00th=[ 1368], 90.00th=[ 1560], 95.00th=[ 1752],
     | 99.00th=[ 2352], 99.50th=[ 3248], 99.90th=[ 5280], 99.95th=[ 6624],
     | 99.99th=[ 8096]
    bw (KB  /s): min=    0, max=56800, per=25.01%, avg=25793.79, stdev=667.75
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.10%, 750=1.87%
    lat (usec) : 1000=17.34%
    lat (msec) : 2=78.57%, 4=1.82%, 10=0.30%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.89%, sys=2.45%, ctx=171220289, majf=0, minf=9245
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=92815677/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=8636: Fri Jan 15 03:22:56 2016
  read : io=723344MB, bw=102876KB/s, iops=12859, runt=7200001msec
    slat (usec): min=2, max=2060, avg=12.02, stdev= 9.16
    clat (usec): min=33, max=76492, avg=1230.74, stdev=375.13
     lat (usec): min=136, max=76498, avg=1242.90, stdev=374.88
    clat percentiles (usec):
     |  1.00th=[  700],  5.00th=[  860], 10.00th=[  948], 20.00th=[ 1004],
     | 30.00th=[ 1096], 40.00th=[ 1128], 50.00th=[ 1176], 60.00th=[ 1240],
     | 70.00th=[ 1272], 80.00th=[ 1368], 90.00th=[ 1560], 95.00th=[ 1752],
     | 99.00th=[ 2352], 99.50th=[ 3248], 99.90th=[ 5216], 99.95th=[ 6560],
     | 99.99th=[ 8160]
    bw (KB  /s): min=    0, max=57520, per=25.01%, avg=25729.72, stdev=637.47
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.08%, 750=1.81%
    lat (usec) : 1000=16.99%
    lat (msec) : 2=79.02%, 4=1.80%, 10=0.30%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.68%, sys=2.40%, ctx=169730779, majf=0, minf=9975
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=92588034/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=1314.3GB, aggrb=191368KB/s, minb=191368KB/s, maxb=191368KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1327.3GB, aggrb=193262KB/s, minb=193262KB/s, maxb=193262KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=725123MB, aggrb=103128KB/s, minb=103128KB/s, maxb=103128KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=723344MB, aggrb=102875KB/s, minb=102875KB/s, maxb=102875KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 03:22:58 AM EST
# 4kB randread with SATA 4 jobs and 2 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 15394 IOPS | 61 MBps | 518 μsec Average Latency 
SATA-R5-1: 15395 IOPS | 61 MBps | 518 μsec Average Latency 
NVMe-R5-0: 17194 IOPS | 68 MBps | 463 μsec Average Latency 
NVMe-R5-1: 17226 IOPS | 68 MBps | 463 μsec Average Latency 

Totals
IOPS: 65209
MBPS: 258
Average Latency (μsec): 490

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=13711: Fri Jan 15 05:24:00 2016
  read : io=432971MB, bw=61578KB/s, iops=15394, runt=7200001msec
    slat (usec): min=1, max=3138, avg=10.81, stdev= 6.55
    clat (usec): min=2, max=58153, avg=507.33, stdev=155.95
     lat (usec): min=108, max=58161, avg=518.29, stdev=155.73
    clat percentiles (usec):
     |  1.00th=[  346],  5.00th=[  374], 10.00th=[  410], 20.00th=[  466],
     | 30.00th=[  478], 40.00th=[  482], 50.00th=[  490], 60.00th=[  494],
     | 70.00th=[  502], 80.00th=[  524], 90.00th=[  604], 95.00th=[  628],
     | 99.00th=[ 1012], 99.50th=[ 1208], 99.90th=[ 2480], 99.95th=[ 2960],
     | 99.99th=[ 6176]
    bw (KB  /s): min=    0, max=16440, per=25.01%, avg=15399.42, stdev=296.65
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%
    lat (usec) : 500=67.13%, 750=30.50%, 1000=1.29%
    lat (msec) : 2=0.91%, 4=0.14%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.84%, sys=2.76%, ctx=204068061, majf=0, minf=6854
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=110840668/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=13715: Fri Jan 15 05:24:00 2016
  read : io=432999MB, bw=61582KB/s, iops=15395, runt=7200001msec
    slat (usec): min=1, max=794, avg= 9.05, stdev= 5.22
    clat (usec): min=14, max=76448, avg=509.13, stdev=167.67
     lat (usec): min=119, max=76457, avg=518.31, stdev=167.52
    clat percentiles (usec):
     |  1.00th=[  346],  5.00th=[  370], 10.00th=[  402], 20.00th=[  466],
     | 30.00th=[  478], 40.00th=[  486], 50.00th=[  490], 60.00th=[  494],
     | 70.00th=[  502], 80.00th=[  516], 90.00th=[  604], 95.00th=[  628],
     | 99.00th=[ 1048], 99.50th=[ 1272], 99.90th=[ 2736], 99.95th=[ 3056],
     | 99.99th=[ 6432]
    bw (KB  /s): min=    0, max=16848, per=25.01%, avg=15400.32, stdev=266.32
    lat (usec) : 20=0.01%, 250=0.02%, 500=68.54%, 750=28.99%, 1000=1.28%
    lat (msec) : 2=0.96%, 4=0.20%, 10=0.02%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.21%, sys=3.16%, ctx=205452453, majf=0, minf=6354
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=110847843/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=13719: Fri Jan 15 05:24:00 2016
  read : io=483606MB, bw=68780KB/s, iops=17194, runt=7200001msec
    slat (usec): min=1, max=3176, avg=10.91, stdev= 6.23
    clat (usec): min=22, max=76361, avg=452.89, stdev=136.75
     lat (usec): min=83, max=76369, avg=463.94, stdev=136.38
    clat percentiles (usec):
     |  1.00th=[  294],  5.00th=[  342], 10.00th=[  354], 20.00th=[  366],
     | 30.00th=[  382], 40.00th=[  426], 50.00th=[  466], 60.00th=[  478],
     | 70.00th=[  486], 80.00th=[  494], 90.00th=[  516], 95.00th=[  604],
     | 99.00th=[  884], 99.50th=[ 1048], 99.90th=[ 1432], 99.95th=[ 1928],
     | 99.99th=[ 5280]
    bw (KB  /s): min=    0, max=18504, per=25.01%, avg=17199.51, stdev=330.98
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.42%, 500=83.50%, 750=14.43%
    lat (usec) : 1000=1.03%
    lat (msec) : 2=0.57%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.91%, sys=3.08%, ctx=228600360, majf=0, minf=6720
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=123803167/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=13723: Fri Jan 15 05:24:00 2016
  read : io=484504MB, bw=68907KB/s, iops=17226, runt=7200001msec
    slat (usec): min=1, max=1384, avg= 9.06, stdev= 5.18
    clat (usec): min=3, max=68029, avg=453.95, stdev=136.56
     lat (usec): min=87, max=68040, avg=463.14, stdev=136.30
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  346], 10.00th=[  354], 20.00th=[  366],
     | 30.00th=[  378], 40.00th=[  430], 50.00th=[  470], 60.00th=[  482],
     | 70.00th=[  490], 80.00th=[  498], 90.00th=[  516], 95.00th=[  604],
     | 99.00th=[  876], 99.50th=[ 1048], 99.90th=[ 1400], 99.95th=[ 1768],
     | 99.99th=[ 5664]
    bw (KB  /s): min=    0, max=18640, per=25.01%, avg=17231.38, stdev=270.77
    lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.45%, 500=83.67%
    lat (usec) : 750=14.27%, 1000=1.00%
    lat (msec) : 2=0.56%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.20%, sys=3.16%, ctx=229949725, majf=0, minf=6733
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=124033120/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=432971MB, aggrb=61578KB/s, minb=61578KB/s, maxb=61578KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=432999MB, aggrb=61582KB/s, minb=61582KB/s, maxb=61582KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=483606MB, aggrb=68779KB/s, minb=68779KB/s, maxb=68779KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=484504MB, aggrb=68907KB/s, minb=68907KB/s, maxb=68907KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 05:24:02 AM EST
# 4kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 30833 IOPS | 123 MBps | 2593 μsec Average Latency 
SATA-R5-1: 30791 IOPS | 123 MBps | 2597 μsec Average Latency 
NVMe-R5-0: 6648 IOPS | 26 MBps | 2405 μsec Average Latency 
NVMe-R5-1: 6796 IOPS | 27 MBps | 2352 μsec Average Latency 

Totals
IOPS: 75068
MBPS: 299
Average Latency (μsec): 2486

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=18865: Fri Jan 15 07:25:04 2016
  read : io=867188MB, bw=123333KB/s, iops=30833, runt=7200002msec
    slat (usec): min=1, max=1731, avg=10.95, stdev= 6.28
    clat (usec): min=2, max=1755.4K, avg=2582.73, stdev=1504.03
     lat (usec): min=170, max=1755.4K, avg=2593.82, stdev=1503.98
    clat percentiles (usec):
     |  1.00th=[ 1832],  5.00th=[ 2064], 10.00th=[ 2160], 20.00th=[ 2288],
     | 30.00th=[ 2384], 40.00th=[ 2480], 50.00th=[ 2512], 60.00th=[ 2608],
     | 70.00th=[ 2736], 80.00th=[ 2864], 90.00th=[ 3024], 95.00th=[ 3248],
     | 99.00th=[ 3760], 99.50th=[ 4128], 99.90th=[ 5600], 99.95th=[ 7008],
     | 99.99th=[10560]
    bw (KB  /s): min=    0, max=97960, per=20.01%, avg=24675.24, stdev=709.08
    lat (usec) : 4=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.02%
    lat (usec) : 1000=0.02%
    lat (msec) : 2=3.54%, 4=95.81%, 10=0.59%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.11%, sys=4.09%, ctx=379978870, majf=0, minf=19716
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=222000089/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=18870: Fri Jan 15 07:25:04 2016
  read : io=865999MB, bw=123164KB/s, iops=30791, runt=7200002msec
    slat (usec): min=1, max=2330, avg=11.03, stdev= 6.33
    clat (usec): min=83, max=1721.6K, avg=2586.29, stdev=1546.08
     lat (usec): min=206, max=1721.6K, avg=2597.46, stdev=1546.02
    clat percentiles (usec):
     |  1.00th=[ 1832],  5.00th=[ 2064], 10.00th=[ 2192], 20.00th=[ 2288],
     | 30.00th=[ 2384], 40.00th=[ 2480], 50.00th=[ 2544], 60.00th=[ 2608],
     | 70.00th=[ 2736], 80.00th=[ 2864], 90.00th=[ 3024], 95.00th=[ 3216],
     | 99.00th=[ 3728], 99.50th=[ 4016], 99.90th=[ 5472], 99.95th=[ 7072],
     | 99.99th=[10816]
    bw (KB  /s): min=    0, max=96320, per=20.01%, avg=24640.72, stdev=714.38
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.02%
    lat (msec) : 2=3.51%, 4=95.92%, 10=0.51%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=1.11%, sys=4.14%, ctx=379290723, majf=0, minf=19759
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=221695725/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=18875: Fri Jan 15 07:25:04 2016
  read : io=186984MB, bw=26593KB/s, iops=6648, runt=7200001msec
    slat (usec): min=1, max=2272, avg=13.41, stdev= 9.41
    clat (usec): min=18, max=1690.8K, avg=2391.44, stdev=1409.10
     lat (usec): min=206, max=1690.8K, avg=2405.03, stdev=1409.11
    clat percentiles (usec):
     |  1.00th=[ 1624],  5.00th=[ 1864], 10.00th=[ 1976], 20.00th=[ 2096],
     | 30.00th=[ 2192], 40.00th=[ 2256], 50.00th=[ 2352], 60.00th=[ 2448],
     | 70.00th=[ 2512], 80.00th=[ 2640], 90.00th=[ 2864], 95.00th=[ 3056],
     | 99.00th=[ 3536], 99.50th=[ 3824], 99.90th=[ 5216], 99.95th=[ 6752],
     | 99.99th=[10304]
    bw (KB  /s): min=    0, max=29552, per=25.02%, avg=6653.92, stdev=207.86
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.03%, 1000=0.04%
    lat (msec) : 2=12.98%, 4=86.60%, 10=0.34%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 2000=0.01%
  cpu          : usr=0.64%, sys=1.73%, ctx=89953518, majf=0, minf=7743
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47867803/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=18879: Fri Jan 15 07:25:04 2016
  read : io=191148MB, bw=27185KB/s, iops=6796, runt=7200001msec
    slat (usec): min=1, max=1639, avg=14.25, stdev=10.14
    clat (usec): min=11, max=1714.3K, avg=2338.23, stdev=1482.31
     lat (usec): min=156, max=1714.4K, avg=2352.67, stdev=1482.31
    clat percentiles (usec):
     |  1.00th=[ 1544],  5.00th=[ 1784], 10.00th=[ 1896], 20.00th=[ 2024],
     | 30.00th=[ 2128], 40.00th=[ 2224], 50.00th=[ 2288], 60.00th=[ 2384],
     | 70.00th=[ 2480], 80.00th=[ 2608], 90.00th=[ 2832], 95.00th=[ 2992],
     | 99.00th=[ 3472], 99.50th=[ 3760], 99.90th=[ 5088], 99.95th=[ 6624],
     | 99.99th=[10304]
    bw (KB  /s): min=    0, max=22792, per=25.02%, avg=6802.00, stdev=210.86
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.03%, 1000=0.04%
    lat (msec) : 2=17.85%, 4=81.78%, 10=0.29%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 2000=0.01%
  cpu          : usr=0.60%, sys=1.68%, ctx=91318174, majf=0, minf=7820
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48933829/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=867188MB, aggrb=123333KB/s, minb=123333KB/s, maxb=123333KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=865999MB, aggrb=123164KB/s, minb=123164KB/s, maxb=123164KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=186984MB, aggrb=26593KB/s, minb=26593KB/s, maxb=26593KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=191148MB, aggrb=27185KB/s, minb=27185KB/s, maxb=27185KB/s, mint=7200001msec, maxt=7200001msec