client3
2016 January 13, 17:05
fio-2.1.10
CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
60 s ramp + 7200 s run, per test; s between tests

# 
# SR-71mach5 Configuration
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 

Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz   Status: Populated, Enabled Core Enabled: 8
Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz   Status: Populated, Enabled Core Enabled: 8
DIMM_A1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 128878AC  
DIMM_B1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 128878B2  
DIMM_C1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 148740E6  
DIMM_D1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13892F10  
DIMM_E1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13A2C785  
DIMM_F1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13892F0E  
DIMM_G1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 128878AD  
DIMM_H1: 8192 MB | 64/72 bits | 1600 MHz | Samsung          | M393B1K70DH0-YK0 | SN: 13892F0A  
____________________________________________________________________
MemTotal:       65701196 kB


# 
# actual mount paramters
# 

10.0.5.144:/control on /root/sr71/control type nfs4 (rw,relatime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=10.0.5.134,local_lock=none,addr=10.0.5.144)
10.10.4.100:/sr71/C0P3 on /root/sr71/SATA0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.4.10,local_lock=none,addr=10.10.4.100)
10.10.5.100:/sr71/C1P3 on /root/sr71/SATA1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.5.10,local_lock=none,addr=10.10.5.100)
10.10.4.100:/sr71/NVMeR5V0P3 on /root/sr71/NVMe0 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.4.10,local_lock=none,addr=10.10.4.100)
10.10.5.100:/sr71/NVMeR5V1P3 on /root/sr71/NVMe1 type nfs4 (rw,noatime,nodiratime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=100,retrans=2,sec=sys,clientaddr=10.10.5.10,local_lock=none,addr=10.10.5.100)



# 
# fio job file
fio-2.1.10
# 

 seconds between fio tests
[global]
ioengine=libaio
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
group_reporting

[SATA-R5-01]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA0/fio.tst

[SATA-R5-11]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/root/sr71/SATA1/fio.tst

[NVMe-R5-01]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe0/fio.tst

[NVMe-R5-11]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/root/sr71/NVMe1/fio.tst




# 
# 201601131705 test details
# 



# 
# Wed 13 Jan 2016 05:05:00 PM EST
# 8kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 8kB randwrite with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 2933 IOPS | 23 MBps | 336 μsec Average Latency 
SATA-R5-1: 2758 IOPS | 22 MBps | 357 μsec Average Latency 
NVMe-R5-0: 5071 IOPS | 40 MBps | 393 μsec Average Latency 
NVMe-R5-1: 5462 IOPS | 43 MBps | 364 μsec Average Latency 

Totals
IOPS: 16224
MBPS: 128
Average Latency (μsec): 362

SATA-R5-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=75030: Wed Jan 13 19:06:01 2016
  write: io=164982MB, bw=23464KB/s, iops=2933, runt=7200001msec
    slat (usec): min=3, max=2278, avg=10.95, stdev= 5.02
    clat (usec): min=0, max=199707, avg=325.29, stdev=306.83
     lat (usec): min=91, max=199722, avg=336.40, stdev=306.20
    clat percentiles (usec):
     |  1.00th=[  102],  5.00th=[  109], 10.00th=[  211], 20.00th=[  258],
     | 30.00th=[  350], 40.00th=[  358], 50.00th=[  358], 60.00th=[  362],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  370], 95.00th=[  386],
     | 99.00th=[  414], 99.50th=[  454], 99.90th=[  510], 99.95th=[  636],
     | 99.99th=[ 1304]
    bw (KB  /s): min=    0, max=41872, per=100.00%, avg=23471.70, stdev=2670.76
    lat (usec) : 2=0.01%, 10=0.01%, 50=0.01%, 100=0.18%, 250=19.42%
    lat (usec) : 500=80.26%, 750=0.09%, 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.78%, sys=2.34%, ctx=42615701, majf=0, minf=1426
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=21117685/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=75031: Wed Jan 13 19:06:01 2016
  write: io=155174MB, bw=22069KB/s, iops=2758, runt=7200001msec
    slat (usec): min=3, max=2604, avg= 9.99, stdev= 4.01
    clat (usec): min=54, max=184754, avg=347.80, stdev=71.33
     lat (usec): min=96, max=184762, avg=357.94, stdev=70.31
    clat percentiles (usec):
     |  1.00th=[  137],  5.00th=[  233], 10.00th=[  302], 20.00th=[  350],
     | 30.00th=[  358], 40.00th=[  358], 50.00th=[  362], 60.00th=[  362],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  370], 95.00th=[  390],
     | 99.00th=[  414], 99.50th=[  462], 99.90th=[  516], 99.95th=[  628],
     | 99.99th=[ 1144]
    bw (KB  /s): min=    0, max=36448, per=100.00%, avg=22075.88, stdev=912.61
    lat (usec) : 100=0.04%, 250=7.45%, 500=92.36%, 750=0.10%, 1000=0.02%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.68%, sys=2.13%, ctx=40099745, majf=0, minf=1306
  IO depths    : 1=100.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19862298/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=75032: Wed Jan 13 19:06:01 2016
  write: io=285294MB, bw=40575KB/s, iops=5071, runt=7200086msec
    slat (usec): min=2, max=991, avg= 9.64, stdev= 3.59
    clat (usec): min=48, max=200964, avg=383.26, stdev=1844.84
     lat (usec): min=94, max=200972, avg=393.05, stdev=1844.81
    clat percentiles (usec):
     |  1.00th=[  181],  5.00th=[  237], 10.00th=[  298], 20.00th=[  354],
     | 30.00th=[  362], 40.00th=[  366], 50.00th=[  366], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  374], 95.00th=[  394],
     | 99.00th=[  418], 99.50th=[  466], 99.90th=[  652], 99.95th=[16064],
     | 99.99th=[92672]
    bw (KB  /s): min=    0, max=51824, per=100.00%, avg=40670.58, stdev=5510.25
    lat (usec) : 50=0.01%, 100=0.02%, 250=6.94%, 500=92.82%, 750=0.13%
    lat (usec) : 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.03%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.18%, sys=3.70%, ctx=72694024, majf=0, minf=1683
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=36517636/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=75033: Wed Jan 13 19:06:01 2016
  write: io=307252MB, bw=43698KB/s, iops=5462, runt=7200001msec
    slat (usec): min=2, max=1177, avg= 9.53, stdev= 3.36
    clat (usec): min=1, max=200278, avg=355.18, stdev=462.77
     lat (usec): min=95, max=200287, avg=364.86, stdev=462.64
    clat percentiles (usec):
     |  1.00th=[  183],  5.00th=[  239], 10.00th=[  310], 20.00th=[  358],
     | 30.00th=[  362], 40.00th=[  366], 50.00th=[  366], 60.00th=[  366],
     | 70.00th=[  366], 80.00th=[  366], 90.00th=[  374], 95.00th=[  394],
     | 99.00th=[  418], 99.50th=[  458], 99.90th=[  524], 99.95th=[  684],
     | 99.99th=[ 1480]
    bw (KB  /s): min=    0, max=47072, per=100.00%, avg=43713.63, stdev=1783.64
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.02%, 250=6.39%, 500=93.42%
    lat (usec) : 750=0.12%, 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.29%, sys=3.90%, ctx=78288276, majf=0, minf=1809
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=39328198/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=164982MB, aggrb=23464KB/s, minb=23464KB/s, maxb=23464KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=155174MB, aggrb=22069KB/s, minb=22069KB/s, maxb=22069KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=285294MB, aggrb=40574KB/s, minb=40574KB/s, maxb=40574KB/s, mint=7200086msec, maxt=7200086msec

Run status group 3 (all jobs):
  WRITE: io=307252MB, aggrb=43697KB/s, minb=43697KB/s, maxb=43697KB/s, mint=7200001msec, maxt=7200001msec



# 
# Wed 13 Jan 2016 07:06:02 PM EST
# 64kB randwrite with SATA 1 jobs and 1 I/Os Outstanding
# 64kB randwrite with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1276 IOPS | 81 MBps | 778 μsec Average Latency 
SATA-R5-1: 1409 IOPS | 90 MBps | 706 μsec Average Latency 
NVMe-R5-0: 603 IOPS | 38 MBps | 13244 μsec Average Latency 
NVMe-R5-1: 651 IOPS | 41 MBps | 12272 μsec Average Latency 

Totals
IOPS: 3939
MBPS: 250
Average Latency (μsec): 6750

SATA-R5-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 10 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=80150: Wed Jan 13 21:07:04 2016
  write: io=574328MB, bw=81682KB/s, iops=1276, runt=7200001msec
    slat (usec): min=11, max=2927, avg=57.37, stdev=13.36
    clat (usec): min=91, max=3949.5K, avg=720.68, stdev=9122.00
     lat (usec): min=201, max=3949.5K, avg=778.41, stdev=9122.07
    clat percentiles (usec):
     |  1.00th=[  294],  5.00th=[  306], 10.00th=[  310], 20.00th=[  322],
     | 30.00th=[  338], 40.00th=[  354], 50.00th=[  422], 60.00th=[  438],
     | 70.00th=[  454], 80.00th=[  524], 90.00th=[  564], 95.00th=[  572],
     | 99.00th=[  692], 99.50th=[  972], 99.90th=[98816], 99.95th=[154624],
     | 99.99th=[382976]
    bw (KB  /s): min=    1, max=164352, per=100.00%, avg=84693.24, stdev=26338.50
    lat (usec) : 100=0.01%, 250=0.03%, 500=78.36%, 750=20.83%, 1000=0.30%
    lat (msec) : 2=0.14%, 4=0.03%, 10=0.02%, 20=0.01%, 50=0.11%
    lat (msec) : 100=0.07%, 250=0.08%, 500=0.02%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.56%, sys=4.01%, ctx=18550545, majf=0, minf=3396
  IO depths    : 1=100.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=9189254/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=80151: Wed Jan 13 21:07:04 2016
  write: io=634288MB, bw=90210KB/s, iops=1409, runt=7200001msec
    slat (usec): min=10, max=283, avg=33.29, stdev= 9.97
    clat (usec): min=127, max=3934.3K, avg=673.46, stdev=8729.93
     lat (usec): min=198, max=3934.4K, avg=706.99, stdev=8729.95
    clat percentiles (usec):
     |  1.00th=[  294],  5.00th=[  310], 10.00th=[  322], 20.00th=[  338],
     | 30.00th=[  342], 40.00th=[  342], 50.00th=[  346], 60.00th=[  350],
     | 70.00th=[  370], 80.00th=[  406], 90.00th=[  462], 95.00th=[  506],
     | 99.00th=[  652], 99.50th=[  908], 99.90th=[96768], 99.95th=[150528],
     | 99.99th=[350208]
    bw (KB  /s): min=    1, max=169088, per=100.00%, avg=93481.13, stdev=31254.94
    lat (usec) : 250=0.03%, 500=94.67%, 750=4.61%, 1000=0.24%
    lat (msec) : 2=0.11%, 4=0.02%, 10=0.01%, 20=0.02%, 50=0.12%
    lat (msec) : 100=0.08%, 250=0.08%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.08%, sys=2.56%, ctx=20483663, majf=0, minf=3394
  IO depths    : 1=100.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10148615/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=80152: Wed Jan 13 21:07:04 2016
  write: io=271729MB, bw=38646KB/s, iops=603, runt=7200016msec
    slat (usec): min=7, max=2985, avg=48.79, stdev=29.79
    clat (usec): min=84, max=3956.1K, avg=13195.92, stdev=52906.37
     lat (usec): min=217, max=3956.1K, avg=13244.98, stdev=52906.03
    clat percentiles (usec):
     |  1.00th=[  298],  5.00th=[  366], 10.00th=[  406], 20.00th=[  454],
     | 30.00th=[  486], 40.00th=[  524], 50.00th=[  556], 60.00th=[  596],
     | 70.00th=[  644], 80.00th=[  716], 90.00th=[ 1064], 95.00th=[126464],
     | 99.00th=[199680], 99.50th=[276480], 99.90th=[464896], 99.95th=[602112],
     | 99.99th=[1056768]
    bw (KB  /s): min=    1, max=129752, per=25.46%, avg=9839.46, stdev=7048.91
    lat (usec) : 100=0.01%, 250=0.27%, 500=34.00%, 750=48.16%, 1000=6.90%
    lat (msec) : 2=2.72%, 4=0.27%, 10=0.12%, 20=0.05%, 50=0.58%
    lat (msec) : 100=1.48%, 250=4.85%, 500=0.52%, 750=0.05%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.15%, sys=0.39%, ctx=8128032, majf=0, minf=18209
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4347666/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=80156: Wed Jan 13 21:07:04 2016
  write: io=293268MB, bw=41705KB/s, iops=651, runt=7200735msec
    slat (usec): min=7, max=275, avg=49.57, stdev=30.69
    clat (usec): min=76, max=3982.5K, avg=12222.98, stdev=50591.19
     lat (usec): min=230, max=3982.5K, avg=12272.84, stdev=50590.77
    clat percentiles (usec):
     |  1.00th=[  290],  5.00th=[  366], 10.00th=[  406], 20.00th=[  458],
     | 30.00th=[  490], 40.00th=[  524], 50.00th=[  564], 60.00th=[  596],
     | 70.00th=[  644], 80.00th=[  716], 90.00th=[ 1020], 95.00th=[95744],
     | 99.00th=[199680], 99.50th=[259072], 99.90th=[452608], 99.95th=[593920],
     | 99.99th=[987136]
    bw (KB  /s): min=    1, max=176384, per=25.58%, avg=10666.96, stdev=7494.31
    lat (usec) : 100=0.01%, 250=0.32%, 500=32.66%, 750=49.78%, 1000=7.02%
    lat (msec) : 2=2.57%, 4=0.28%, 10=0.11%, 20=0.05%, 50=0.69%
    lat (msec) : 100=1.67%, 250=4.32%, 500=0.46%, 750=0.05%, 1000=0.02%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.15%, sys=0.43%, ctx=8796313, majf=0, minf=20601
  IO depths    : 1=0.1%, 2=101.1%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=4692287/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=574328MB, aggrb=81682KB/s, minb=81682KB/s, maxb=81682KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=634288MB, aggrb=90209KB/s, minb=90209KB/s, maxb=90209KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=271729MB, aggrb=38645KB/s, minb=38645KB/s, maxb=38645KB/s, mint=7200016msec, maxt=7200016msec

Run status group 3 (all jobs):
  WRITE: io=293268MB, aggrb=41704KB/s, minb=41704KB/s, maxb=41704KB/s, mint=7200735msec, maxt=7200735msec



# 
# Wed 13 Jan 2016 09:07:07 PM EST
# 8kB randrw with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randrw with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 5963 IOPS | 31 MBps | 7163 μsec Average Latency 
SATA-R5-1: 5846 IOPS | 31 MBps | 7317 μsec Average Latency 
NVMe-R5-0: 10027 IOPS | 53 MBps | 4200 μsec Average Latency 
NVMe-R5-1: 10696 IOPS | 57 MBps | 3909 μsec Average Latency 

Totals
IOPS: 32532
MBPS: 172
Average Latency (μsec): 5647

SATA-R5-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=85315: Wed Jan 13 23:08:08 2016
  read : io=224764MB, bw=31966KB/s, iops=3995, runt=7200055msec
    slat (usec): min=1, max=772, avg=19.24, stdev=19.51
    clat (usec): min=16, max=1611.4K, avg=1849.34, stdev=4860.33
     lat (usec): min=148, max=1611.5K, avg=1868.78, stdev=4861.09
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  454], 10.00th=[  498], 20.00th=[  564],
     | 30.00th=[  612], 40.00th=[  660], 50.00th=[  724], 60.00th=[  796],
     | 70.00th=[  956], 80.00th=[ 1704], 90.00th=[ 3120], 95.00th=[ 9664],
     | 99.00th=[16768], 99.50th=[18304], 99.90th=[25216], 99.95th=[28032],
     | 99.99th=[78336]
    bw (KB  /s): min=    0, max=28176, per=25.28%, avg=8081.97, stdev=8555.93
  write: io=110702MB, bw=15744KB/s, iops=1968, runt=7200055msec
    slat (usec): min=2, max=1361, avg=24.73, stdev=22.55
    clat (usec): min=20, max=1662.7K, avg=12433.87, stdev=32894.21
     lat (usec): min=139, max=1662.7K, avg=12458.80, stdev=32895.00
    clat percentiles (usec):
     |  1.00th=[  382],  5.00th=[  596], 10.00th=[ 1368], 20.00th=[ 2640],
     | 30.00th=[ 3248], 40.00th=[ 3760], 50.00th=[ 4256], 60.00th=[ 4768],
     | 70.00th=[ 5600], 80.00th=[ 7328], 90.00th=[16768], 95.00th=[60160],
     | 99.00th=[185344], 99.50th=[209920], 99.90th=[264192], 99.95th=[305152],
     | 99.99th=[618496]
    bw (KB  /s): min=    0, max=13392, per=25.29%, avg=3980.92, stdev=4211.44
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.05%, 500=7.97%
    lat (usec) : 750=30.96%, 1000=11.87%
    lat (msec) : 2=8.36%, 4=17.70%, 10=15.41%, 20=4.37%, 50=1.48%
    lat (msec) : 100=0.71%, 250=1.08%, 500=0.04%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.58%, sys=1.64%, ctx=72311708, majf=0, minf=15271
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=28769780/w=14169788/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=85319: Wed Jan 13 23:08:08 2016
  read : io=220361MB, bw=31340KB/s, iops=3917, runt=7200086msec
    slat (usec): min=1, max=2003, avg=18.18, stdev=18.69
    clat (usec): min=1, max=1604.9K, avg=1867.75, stdev=5053.13
     lat (usec): min=122, max=1604.9K, avg=1886.13, stdev=5053.76
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  454], 10.00th=[  494], 20.00th=[  564],
     | 30.00th=[  612], 40.00th=[  660], 50.00th=[  716], 60.00th=[  788],
     | 70.00th=[  948], 80.00th=[ 1720], 90.00th=[ 3120], 95.00th=[ 9792],
     | 99.00th=[17280], 99.50th=[18816], 99.90th=[25472], 99.95th=[28032],
     | 99.99th=[58624]
    bw (KB  /s): min=    0, max=26096, per=25.29%, avg=7925.31, stdev=8506.27
  write: io=108538MB, bw=15436KB/s, iops=1929, runt=7200086msec
    slat (usec): min=2, max=1580, avg=23.31, stdev=21.63
    clat (usec): min=23, max=1726.7K, avg=12724.86, stdev=33827.40
     lat (usec): min=135, max=1726.8K, avg=12748.36, stdev=33828.08
    clat percentiles (usec):
     |  1.00th=[  378],  5.00th=[  596], 10.00th=[ 1384], 20.00th=[ 2640],
     | 30.00th=[ 3248], 40.00th=[ 3760], 50.00th=[ 4256], 60.00th=[ 4832],
     | 70.00th=[ 5664], 80.00th=[ 7456], 90.00th=[17536], 95.00th=[62208],
     | 99.00th=[191488], 99.50th=[214016], 99.90th=[264192], 99.95th=[292864],
     | 99.99th=[634880]
    bw (KB  /s): min=    0, max=12640, per=25.29%, avg=3903.79, stdev=4189.53
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.05%
    lat (usec) : 500=8.21%, 750=31.25%, 1000=11.43%
    lat (msec) : 2=8.16%, 4=17.76%, 10=15.31%, 20=4.43%, 50=1.54%
    lat (msec) : 100=0.69%, 250=1.12%, 500=0.04%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.54%, sys=1.55%, ctx=71182364, majf=0, minf=15342
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=28206196/w=13892872/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=85323: Wed Jan 13 23:08:08 2016
  read : io=377920MB, bw=53749KB/s, iops=6718, runt=7200002msec
    slat (usec): min=2, max=1403, avg=19.19, stdev=18.50
    clat (usec): min=16, max=1597.8K, avg=1204.14, stdev=3625.07
     lat (usec): min=114, max=1597.9K, avg=1223.52, stdev=3625.71
    clat percentiles (usec):
     |  1.00th=[  326],  5.00th=[  406], 10.00th=[  454], 20.00th=[  506],
     | 30.00th=[  564], 40.00th=[  612], 50.00th=[  684], 60.00th=[  788],
     | 70.00th=[ 1112], 80.00th=[ 1944], 90.00th=[ 2704], 95.00th=[ 3408],
     | 99.00th=[ 4832], 99.50th=[ 5472], 99.90th=[ 7520], 99.95th=[ 8512],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=36720, per=25.14%, avg=13513.28, stdev=9486.87
  write: io=186170MB, bw=26478KB/s, iops=3309, runt=7200002msec
    slat (usec): min=3, max=1723, avg=24.36, stdev=21.20
    clat (usec): min=1, max=1601.5K, avg=7153.59, stdev=11573.31
     lat (usec): min=125, max=1601.6K, avg=7178.15, stdev=11575.13
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  532], 10.00th=[  924], 20.00th=[ 1816],
     | 30.00th=[ 2256], 40.00th=[ 2640], 50.00th=[ 3056], 60.00th=[ 3696],
     | 70.00th=[ 6624], 80.00th=[13248], 90.00th=[20352], 95.00th=[25216],
     | 99.00th=[33536], 99.50th=[36608], 99.90th=[43776], 99.95th=[49408],
     | 99.99th=[468992]
    bw (KB  /s): min=    9, max=17536, per=25.15%, avg=6657.97, stdev=4673.88
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.15%
    lat (usec) : 500=13.95%, 750=27.15%, 1000=7.58%
    lat (msec) : 2=12.98%, 4=23.88%, 10=6.12%, 20=4.77%, 50=3.41%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.97%, sys=2.84%, ctx=121197179, majf=0, minf=15006
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48373811/w=23829750/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=85327: Wed Jan 13 23:08:08 2016
  read : io=403105MB, bw=57330KB/s, iops=7166, runt=7200007msec
    slat (usec): min=2, max=2557, avg=20.29, stdev=19.72
    clat (usec): min=1, max=1611.4K, avg=1181.66, stdev=3742.33
     lat (usec): min=109, max=1611.4K, avg=1202.15, stdev=3742.91
    clat percentiles (usec):
     |  1.00th=[  326],  5.00th=[  406], 10.00th=[  454], 20.00th=[  510],
     | 30.00th=[  572], 40.00th=[  620], 50.00th=[  692], 60.00th=[  804],
     | 70.00th=[ 1144], 80.00th=[ 1928], 90.00th=[ 2576], 95.00th=[ 3152],
     | 99.00th=[ 4576], 99.50th=[ 5088], 99.90th=[ 7264], 99.95th=[ 8256],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=36112, per=25.14%, avg=14415.21, stdev=9219.91
  write: io=198581MB, bw=28243KB/s, iops=3530, runt=7200007msec
    slat (usec): min=2, max=740, avg=25.92, stdev=22.51
    clat (usec): min=15, max=1617.3K, avg=6590.59, stdev=10776.96
     lat (usec): min=121, max=1617.3K, avg=6616.73, stdev=10778.60
    clat percentiles (usec):
     |  1.00th=[  354],  5.00th=[  540], 10.00th=[  956], 20.00th=[ 1864],
     | 30.00th=[ 2320], 40.00th=[ 2736], 50.00th=[ 3184], 60.00th=[ 3952],
     | 70.00th=[ 7008], 80.00th=[11968], 90.00th=[17536], 95.00th=[21376],
     | 99.00th=[28288], 99.50th=[30848], 99.90th=[37120], 99.95th=[42240],
     | 99.99th=[440320]
    bw (KB  /s): min=   15, max=17904, per=25.15%, avg=7102.38, stdev=4543.56
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.16%
    lat (usec) : 500=13.54%, 750=26.72%, 1000=7.87%
    lat (msec) : 2=13.46%, 4=23.72%, 10=6.66%, 20=5.71%, 50=2.13%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.10%, sys=3.10%, ctx=128221957, majf=0, minf=15321
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=51597395/w=25418321/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=224764MB, aggrb=31966KB/s, minb=31966KB/s, maxb=31966KB/s, mint=7200055msec, maxt=7200055msec
  WRITE: io=110702MB, aggrb=15744KB/s, minb=15744KB/s, maxb=15744KB/s, mint=7200055msec, maxt=7200055msec

Run status group 1 (all jobs):
   READ: io=220361MB, aggrb=31339KB/s, minb=31339KB/s, maxb=31339KB/s, mint=7200086msec, maxt=7200086msec
  WRITE: io=108538MB, aggrb=15436KB/s, minb=15436KB/s, maxb=15436KB/s, mint=7200086msec, maxt=7200086msec

Run status group 2 (all jobs):
   READ: io=377920MB, aggrb=53748KB/s, minb=53748KB/s, maxb=53748KB/s, mint=7200002msec, maxt=7200002msec
  WRITE: io=186170MB, aggrb=26477KB/s, minb=26477KB/s, maxb=26477KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=403105MB, aggrb=57330KB/s, minb=57330KB/s, maxb=57330KB/s, mint=7200007msec, maxt=7200007msec
  WRITE: io=198581MB, aggrb=28242KB/s, minb=28242KB/s, maxb=28242KB/s, mint=7200007msec, maxt=7200007msec



# 
# Wed 13 Jan 2016 11:08:11 PM EST
# 64kB randrw with SATA 5 jobs and 4 I/Os Outstanding
# 64kB randrw with NVMe 5 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1851 IOPS | 79 MBps | 14604 μsec Average Latency 
SATA-R5-1: 1795 IOPS | 77 MBps | 15072 μsec Average Latency 
NVMe-R5-0: 1533 IOPS | 65 MBps | 7790 μsec Average Latency 
NVMe-R5-1: 1637 IOPS | 70 MBps | 7293 μsec Average Latency 

Totals
IOPS: 6816
MBPS: 291
Average Latency (μsec): 11189

SATA-R5-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=90623: Thu Jan 14 01:09:12 2016
  read : io=558650MB, bw=79452KB/s, iops=1241, runt=7200013msec
    slat (usec): min=5, max=1813, avg=34.78, stdev=24.60
    clat (usec): min=146, max=4119.3K, avg=3382.98, stdev=31399.06
     lat (usec): min=228, max=4119.4K, avg=3418.00, stdev=31399.13
    clat percentiles (usec):
     |  1.00th=[  748],  5.00th=[  900], 10.00th=[  988], 20.00th=[ 1096],
     | 30.00th=[ 1192], 40.00th=[ 1288], 50.00th=[ 1368], 60.00th=[ 1496],
     | 70.00th=[ 1672], 80.00th=[ 2608], 90.00th=[ 5024], 95.00th=[ 7712],
     | 99.00th=[17536], 99.50th=[42240], 99.90th=[203776], 99.95th=[602112],
     | 99.99th=[1630208]
    bw (KB  /s): min=    1, max=51072, per=23.35%, avg=18549.03, stdev=12489.45
  write: io=274790MB, bw=39081KB/s, iops=610, runt=7200013msec
    slat (usec): min=9, max=2821, avg=57.51, stdev=32.95
    clat (usec): min=73, max=4314.3K, avg=25732.97, stdev=100116.11
     lat (usec): min=240, max=4314.4K, avg=25790.76, stdev=100116.13
    clat percentiles (usec):
     |  1.00th=[  502],  5.00th=[ 1128], 10.00th=[ 4896], 20.00th=[ 7328],
     | 30.00th=[ 8640], 40.00th=[ 9536], 50.00th=[10560], 60.00th=[11584],
     | 70.00th=[13760], 80.00th=[18560], 90.00th=[47360], 95.00th=[69120],
     | 99.00th=[268288], 99.50th=[528384], 99.90th=[1630208], 99.95th=[2007040],
     | 99.99th=[2801664]
    bw (KB  /s): min=    1, max=23888, per=23.35%, avg=9126.88, stdev=6094.07
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.33%, 750=1.05%, 1000=7.05%
    lat (msec) : 2=45.16%, 4=7.72%, 10=18.56%, 20=13.53%, 50=3.25%
    lat (msec) : 100=2.09%, 250=0.86%, 500=0.20%, 750=0.06%, 1000=0.04%
    lat (msec) : 2000=0.09%, >=2000=0.02%
  cpu          : usr=0.39%, sys=0.89%, ctx=25465968, majf=0, minf=49025
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8938400/w=4396619/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=90628: Thu Jan 14 01:09:12 2016
  read : io=541722MB, bw=77045KB/s, iops=1203, runt=7200016msec
    slat (usec): min=5, max=2815, avg=34.21, stdev=24.24
    clat (usec): min=167, max=4334.1K, avg=3467.14, stdev=31939.71
     lat (usec): min=217, max=4335.2K, avg=3501.62, stdev=31939.80
    clat percentiles (usec):
     |  1.00th=[  732],  5.00th=[  876], 10.00th=[  964], 20.00th=[ 1080],
     | 30.00th=[ 1176], 40.00th=[ 1272], 50.00th=[ 1368], 60.00th=[ 1496],
     | 70.00th=[ 1720], 80.00th=[ 2896], 90.00th=[ 5408], 95.00th=[ 8032],
     | 99.00th=[16768], 99.50th=[42240], 99.90th=[203776], 99.95th=[618496],
     | 99.99th=[1646592]
    bw (KB  /s): min=    1, max=50816, per=23.31%, avg=17960.81, stdev=12333.01
  write: io=266464MB, bw=37897KB/s, iops=592, runt=7200016msec
    slat (usec): min=9, max=816, avg=56.59, stdev=32.53
    clat (usec): min=74, max=4373.4K, avg=26587.84, stdev=101283.01
     lat (usec): min=241, max=4373.5K, avg=26644.73, stdev=101283.04
    clat percentiles (usec):
     |  1.00th=[  494],  5.00th=[ 1112], 10.00th=[ 4768], 20.00th=[ 7328],
     | 30.00th=[ 8640], 40.00th=[ 9536], 50.00th=[10560], 60.00th=[11840],
     | 70.00th=[14272], 80.00th=[20096], 90.00th=[50944], 95.00th=[70144],
     | 99.00th=[276480], 99.50th=[561152], 99.90th=[1646592], 99.95th=[2023424],
     | 99.99th=[2899968]
    bw (KB  /s): min=    1, max=23888, per=23.33%, avg=8840.11, stdev=6027.26
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.35%, 750=1.23%, 1000=7.98%
    lat (msec) : 2=43.01%, 4=7.96%, 10=19.19%, 20=13.10%, 50=3.52%
    lat (msec) : 100=2.39%, 250=0.84%, 500=0.21%, 750=0.06%, 1000=0.05%
    lat (msec) : 2000=0.09%, >=2000=0.02%
  cpu          : usr=0.36%, sys=0.86%, ctx=24775759, majf=0, minf=47443
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8667551/w=4263404/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=90633: Thu Jan 14 01:09:12 2016
  read : io=462600MB, bw=65791KB/s, iops=1027, runt=7200069msec
    slat (usec): min=5, max=876, avg=38.56, stdev=26.96
    clat (usec): min=1, max=4152.1K, avg=3995.25, stdev=35924.14
     lat (usec): min=179, max=4152.1K, avg=4034.10, stdev=35924.19
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  716], 10.00th=[  820], 20.00th=[ 1004],
     | 30.00th=[ 1256], 40.00th=[ 1864], 50.00th=[ 2544], 60.00th=[ 3056],
     | 70.00th=[ 3600], 80.00th=[ 4320], 90.00th=[ 5152], 95.00th=[ 5920],
     | 99.00th=[ 8384], 99.50th=[10944], 99.90th=[272384], 99.95th=[815104],
     | 99.99th=[1777664]
    bw (KB  /s): min=    1, max=67456, per=23.47%, avg=15438.52, stdev=10166.81
  write: io=227789MB, bw=32396KB/s, iops=506, runt=7200069msec
    slat (usec): min=8, max=1040, avg=61.81, stdev=35.87
    clat (usec): min=1, max=4159.5K, avg=11485.84, stdev=66478.83
     lat (usec): min=213, max=4159.6K, avg=11547.99, stdev=66478.68
    clat percentiles (usec):
     |  1.00th=[  438],  5.00th=[  756], 10.00th=[ 1004], 20.00th=[ 1640],
     | 30.00th=[ 2512], 40.00th=[ 4256], 50.00th=[ 6688], 60.00th=[ 9024],
     | 70.00th=[10944], 80.00th=[12864], 90.00th=[15296], 95.00th=[17280],
     | 99.00th=[89600], 99.50th=[160768], 99.90th=[1171456], 99.95th=[1564672],
     | 99.99th=[2408448]
    bw (KB  /s): min=    1, max=33664, per=23.47%, avg=7603.02, stdev=5002.86
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.02%, 500=0.83%, 750=5.13%, 1000=10.66%
    lat (msec) : 2=19.44%, 4=27.60%, 10=24.38%, 20=10.90%, 50=0.41%
    lat (msec) : 100=0.10%, 250=0.33%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.06%, >=2000=0.01%
  cpu          : usr=0.36%, sys=0.80%, ctx=21473644, majf=0, minf=28867
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7401599/w=3644627/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=90638: Thu Jan 14 01:09:12 2016
  read : io=494024MB, bw=70261KB/s, iops=1097, runt=7200022msec
    slat (usec): min=6, max=1921, avg=38.89, stdev=27.11
    clat (usec): min=1, max=4151.9K, avg=3747.05, stdev=34839.34
     lat (usec): min=189, max=4151.1K, avg=3786.28, stdev=34839.38
    clat percentiles (usec):
     |  1.00th=[  556],  5.00th=[  716], 10.00th=[  820], 20.00th=[ 1004],
     | 30.00th=[ 1240], 40.00th=[ 1704], 50.00th=[ 2352], 60.00th=[ 2832],
     | 70.00th=[ 3312], 80.00th=[ 3952], 90.00th=[ 4768], 95.00th=[ 5408],
     | 99.00th=[ 7904], 99.50th=[10176], 99.90th=[250880], 99.95th=[757760],
     | 99.99th=[1728512]
    bw (KB  /s): min=   23, max=72192, per=23.48%, avg=16499.21, stdev=10284.83
  write: io=243124MB, bw=34578KB/s, iops=540, runt=7200022msec
    slat (usec): min=9, max=1290, avg=62.23, stdev=35.78
    clat (usec): min=1, max=4155.6K, avg=10737.68, stdev=64552.27
     lat (usec): min=228, max=4155.7K, avg=10800.30, stdev=64552.14
    clat percentiles (usec):
     |  1.00th=[  434],  5.00th=[  756], 10.00th=[ 1020], 20.00th=[ 1688],
     | 30.00th=[ 2512], 40.00th=[ 3984], 50.00th=[ 6112], 60.00th=[ 8160],
     | 70.00th=[ 9920], 80.00th=[11712], 90.00th=[14016], 95.00th=[15936],
     | 99.00th=[76288], 99.50th=[152576], 99.90th=[1138688], 99.95th=[1531904],
     | 99.99th=[2408448]
    bw (KB  /s): min=    1, max=37248, per=23.49%, avg=8121.18, stdev=5062.61
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.02%, 500=0.88%, 750=5.05%, 1000=10.52%
    lat (msec) : 2=21.16%, 4=29.61%, 10=22.59%, 20=9.35%, 50=0.21%
    lat (msec) : 100=0.10%, 250=0.32%, 500=0.06%, 750=0.03%, 1000=0.03%
    lat (msec) : 2000=0.05%, >=2000=0.01%
  cpu          : usr=0.39%, sys=0.85%, ctx=22911065, majf=0, minf=27737
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=7904377/w=3889986/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=558650MB, aggrb=79452KB/s, minb=79452KB/s, maxb=79452KB/s, mint=7200013msec, maxt=7200013msec
  WRITE: io=274790MB, aggrb=39081KB/s, minb=39081KB/s, maxb=39081KB/s, mint=7200013msec, maxt=7200013msec

Run status group 1 (all jobs):
   READ: io=541722MB, aggrb=77044KB/s, minb=77044KB/s, maxb=77044KB/s, mint=7200016msec, maxt=7200016msec
  WRITE: io=266464MB, aggrb=37896KB/s, minb=37896KB/s, maxb=37896KB/s, mint=7200016msec, maxt=7200016msec

Run status group 2 (all jobs):
   READ: io=462600MB, aggrb=65791KB/s, minb=65791KB/s, maxb=65791KB/s, mint=7200069msec, maxt=7200069msec
  WRITE: io=227789MB, aggrb=32396KB/s, minb=32396KB/s, maxb=32396KB/s, mint=7200069msec, maxt=7200069msec

Run status group 3 (all jobs):
   READ: io=494024MB, aggrb=70260KB/s, minb=70260KB/s, maxb=70260KB/s, mint=7200022msec, maxt=7200022msec
  WRITE: io=243124MB, aggrb=34577KB/s, minb=34577KB/s, maxb=34577KB/s, mint=7200022msec, maxt=7200022msec



# 
# Thu 14 Jan 2016 01:09:15 AM EST
# 1mB write with SATA 1 jobs and 2 I/Os Outstanding
# 1mB write with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 336 IOPS | 344 MBps | 5940 μsec Average Latency 
SATA-R5-1: 309 IOPS | 316 MBps | 6460 μsec Average Latency 
NVMe-R5-0: 139 IOPS | 142 MBps | 57470 μsec Average Latency 
NVMe-R5-1: 136 IOPS | 139 MBps | 58550 μsec Average Latency 

Totals
IOPS: 920
MBPS: 941
Average Latency (μsec): 32105

SATA-R5-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
SATA-R5-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-01: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=96560: Thu Jan 14 03:10:15 2016
  write: io=2364.9GB, bw=344396KB/s, iops=336, runt=7200002msec
    slat (usec): min=102, max=2824, avg=207.41, stdev=45.54
    clat (msec): min=1, max=650, avg= 5.74, stdev=14.34
     lat (msec): min=1, max=650, avg= 5.94, stdev=14.34
    clat percentiles (usec):
     |  1.00th=[ 1384],  5.00th=[ 1640], 10.00th=[ 1832], 20.00th=[ 1928],
     | 30.00th=[ 1992], 40.00th=[ 2064], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2384], 80.00th=[ 2960], 90.00th=[ 6880], 95.00th=[37120],
     | 99.00th=[69120], 99.50th=[85504], 99.90th=[136192], 99.95th=[179200],
     | 99.99th=[272384]
    bw (KB  /s): min=   17, max=815520, per=100.00%, avg=346422.67, stdev=118476.23
    lat (msec) : 2=31.47%, 4=54.04%, 10=7.96%, 20=0.72%, 50=2.88%
    lat (msec) : 100=2.66%, 250=0.28%, 500=0.01%, 750=0.01%
  cpu          : usr=1.52%, sys=3.23%, ctx=11545837, majf=0, minf=67241
  IO depths    : 1=0.1%, 2=100.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2421536/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=96561: Thu Jan 14 03:10:15 2016
  write: io=2175.1GB, bw=316894KB/s, iops=309, runt=7200002msec
    slat (usec): min=103, max=1120, avg=264.66, stdev=63.98
    clat (msec): min=1, max=547, avg= 6.19, stdev=15.53
     lat (msec): min=1, max=547, avg= 6.46, stdev=15.53
    clat percentiles (usec):
     |  1.00th=[ 1432],  5.00th=[ 1704], 10.00th=[ 1832], 20.00th=[ 1912],
     | 30.00th=[ 1992], 40.00th=[ 2040], 50.00th=[ 2096], 60.00th=[ 2192],
     | 70.00th=[ 2384], 80.00th=[ 2928], 90.00th=[ 7328], 95.00th=[41216],
     | 99.00th=[74240], 99.50th=[89600], 99.90th=[148480], 99.95th=[197632],
     | 99.99th=[292864]
    bw (KB  /s): min=   17, max=813477, per=100.00%, avg=318904.95, stdev=110307.41
    lat (msec) : 2=32.25%, 4=52.74%, 10=7.81%, 20=0.74%, 50=2.86%
    lat (msec) : 100=3.28%, 250=0.31%, 500=0.02%, 750=0.01%
  cpu          : usr=1.56%, sys=3.43%, ctx=10465867, majf=0, minf=66202
  IO depths    : 1=0.1%, 2=100.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=2228162/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=96562: Thu Jan 14 03:10:15 2016
  write: io=978.74GB, bw=142533KB/s, iops=139, runt=7200122msec
    slat (usec): min=107, max=1478, avg=329.05, stdev=89.59
    clat (msec): min=1, max=1451, avg=57.14, stdev=72.78
     lat (msec): min=1, max=1452, avg=57.47, stdev=72.77
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   11], 60.00th=[   57],
     | 70.00th=[   77], 80.00th=[  106], 90.00th=[  157], 95.00th=[  200],
     | 99.00th=[  289], 99.50th=[  379], 99.90th=[  594], 99.95th=[  635],
     | 99.99th=[  865]
    bw (KB  /s): min=   17, max=795081, per=100.00%, avg=143904.82, stdev=53071.06
    lat (msec) : 2=0.51%, 4=7.63%, 10=40.90%, 20=4.66%, 50=4.03%
    lat (msec) : 100=20.72%, 250=19.79%, 500=1.62%, 750=0.11%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.04%, sys=1.80%, ctx=4578787, majf=0, minf=70386
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.5%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=1002198/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=96563: Thu Jan 14 03:10:15 2016
  write: io=983764MB, bw=139913KB/s, iops=136, runt=7200017msec
    slat (usec): min=101, max=1629, avg=329.81, stdev=86.32
    clat (msec): min=1, max=1242, avg=58.22, stdev=73.29
     lat (msec): min=1, max=1243, avg=58.55, stdev=73.28
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    4], 10.00th=[    5], 20.00th=[    7],
     | 30.00th=[    8], 40.00th=[    9], 50.00th=[   11], 60.00th=[   58],
     | 70.00th=[   79], 80.00th=[  110], 90.00th=[  161], 95.00th=[  204],
     | 99.00th=[  293], 99.50th=[  371], 99.90th=[  545], 99.95th=[  611],
     | 99.99th=[  816]
    bw (KB  /s): min=   17, max=758291, per=100.00%, avg=141132.25, stdev=51277.01
    lat (msec) : 2=0.55%, 4=8.11%, 10=40.14%, 20=4.60%, 50=3.85%
    lat (msec) : 100=20.56%, 250=20.31%, 500=1.74%, 750=0.11%, 1000=0.02%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.04%, sys=1.77%, ctx=4491555, majf=0, minf=65661
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.5%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=983757/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
  WRITE: io=2364.9GB, aggrb=344396KB/s, minb=344396KB/s, maxb=344396KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
  WRITE: io=2175.1GB, aggrb=316894KB/s, minb=316894KB/s, maxb=316894KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=978.74GB, aggrb=142533KB/s, minb=142533KB/s, maxb=142533KB/s, mint=7200122msec, maxt=7200122msec

Run status group 3 (all jobs):
  WRITE: io=983764MB, aggrb=139912KB/s, minb=139912KB/s, maxb=139912KB/s, mint=7200017msec, maxt=7200017msec



# 
# Thu 14 Jan 2016 03:10:18 AM EST
# 1mB read with SATA 1 jobs and 8 I/Os Outstanding
# 1mB read with NVMe 1 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 707 IOPS | 724 MBps | 11310 μsec Average Latency 
SATA-R5-1: 707 IOPS | 724 MBps | 11310 μsec Average Latency 
NVMe-R5-0: 331 IOPS | 339 MBps | 6030 μsec Average Latency 
NVMe-R5-1: 339 IOPS | 347 MBps | 5890 μsec Average Latency 

Totals
IOPS: 2084
MBPS: 2134
Average Latency (μsec): 8635

SATA-R5-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
NVMe-R5-11: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=101685: Thu Jan 14 05:11:18 2016
  read : io=4971.9GB, bw=724067KB/s, iops=707, runt=7200010msec
    slat (usec): min=86, max=4558, avg=246.68, stdev=91.02
    clat (msec): min=1, max=37, avg=11.06, stdev= 2.61
     lat (msec): min=2, max=37, avg=11.31, stdev= 2.61
    clat percentiles (usec):
     |  1.00th=[ 5408],  5.00th=[ 7328], 10.00th=[ 8384], 20.00th=[ 9408],
     | 30.00th=[ 9920], 40.00th=[10304], 50.00th=[10560], 60.00th=[10944],
     | 70.00th=[11584], 80.00th=[12736], 90.00th=[14656], 95.00th=[16320],
     | 99.00th=[18816], 99.50th=[19840], 99.90th=[22656], 99.95th=[23936],
     | 99.99th=[27008]
    bw (KB  /s): min=   17, max=829828, per=100.00%, avg=724691.35, stdev=33069.06
    lat (msec) : 2=0.01%, 4=0.07%, 10=32.96%, 20=66.52%, 50=0.45%
  cpu          : usr=0.44%, sys=9.05%, ctx=24662344, majf=0, minf=67226
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5091098/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=101686: Thu Jan 14 05:11:18 2016
  read : io=4972.4GB, bw=724154KB/s, iops=707, runt=7200008msec
    slat (usec): min=74, max=10378, avg=198.62, stdev=102.63
    clat (msec): min=1, max=36, avg=11.11, stdev= 2.61
     lat (msec): min=1, max=36, avg=11.31, stdev= 2.61
    clat percentiles (usec):
     |  1.00th=[ 5472],  5.00th=[ 7456], 10.00th=[ 8512], 20.00th=[ 9408],
     | 30.00th=[ 9920], 40.00th=[10304], 50.00th=[10688], 60.00th=[10944],
     | 70.00th=[11584], 80.00th=[12864], 90.00th=[14784], 95.00th=[16320],
     | 99.00th=[18816], 99.50th=[19840], 99.90th=[22656], 99.95th=[23936],
     | 99.99th=[26752]
    bw (KB  /s): min=   17, max=811008, per=100.00%, avg=724781.95, stdev=33095.28
    lat (msec) : 2=0.01%, 4=0.08%, 10=32.04%, 20=67.42%, 50=0.46%
  cpu          : usr=0.40%, sys=8.93%, ctx=24990524, majf=0, minf=69328
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5091705/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=101687: Thu Jan 14 05:11:18 2016
  read : io=2330.6GB, bw=339412KB/s, iops=331, runt=7200008msec
    slat (usec): min=80, max=5933, avg=225.86, stdev=87.75
    clat (usec): min=862, max=29528, avg=5803.54, stdev=2624.99
     lat (msec): min=1, max=29, avg= 6.03, stdev= 2.63
    clat percentiles (usec):
     |  1.00th=[ 1464],  5.00th=[ 2064], 10.00th=[ 2928], 20.00th=[ 4128],
     | 30.00th=[ 4640], 40.00th=[ 5024], 50.00th=[ 5280], 60.00th=[ 5664],
     | 70.00th=[ 6112], 80.00th=[ 7008], 90.00th=[ 9792], 95.00th=[11584],
     | 99.00th=[13888], 99.50th=[14784], 99.90th=[17024], 99.95th=[18048],
     | 99.99th=[20608]
    bw (KB  /s): min=   17, max=411648, per=100.00%, avg=339732.65, stdev=23146.23
    lat (usec) : 1000=0.01%
    lat (msec) : 2=4.53%, 4=13.85%, 10=72.13%, 20=9.47%, 50=0.02%
  cpu          : usr=0.23%, sys=3.63%, ctx=11958696, majf=0, minf=67252
  IO depths    : 1=0.1%, 2=100.7%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2386492/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=101688: Thu Jan 14 05:11:18 2016
  read : io=2385.4GB, bw=347389KB/s, iops=339, runt=7200008msec
    slat (usec): min=80, max=6000, avg=231.10, stdev=99.44
    clat (usec): min=738, max=31491, avg=5659.89, stdev=2547.93
     lat (msec): min=1, max=31, avg= 5.89, stdev= 2.55
    clat percentiles (usec):
     |  1.00th=[ 1496],  5.00th=[ 2064], 10.00th=[ 2800], 20.00th=[ 3984],
     | 30.00th=[ 4576], 40.00th=[ 4960], 50.00th=[ 5216], 60.00th=[ 5536],
     | 70.00th=[ 5984], 80.00th=[ 6752], 90.00th=[ 9536], 95.00th=[11200],
     | 99.00th=[13632], 99.50th=[14400], 99.90th=[16768], 99.95th=[17792],
     | 99.99th=[20608]
    bw (KB  /s): min=   17, max=407552, per=100.00%, avg=347717.86, stdev=21874.75
    lat (usec) : 750=0.01%, 1000=0.01%
    lat (msec) : 2=4.44%, 4=15.91%, 10=70.97%, 20=8.66%, 50=0.01%
  cpu          : usr=0.23%, sys=3.73%, ctx=12239944, majf=0, minf=67904
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2442580/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=4971.9GB, aggrb=724067KB/s, minb=724067KB/s, maxb=724067KB/s, mint=7200010msec, maxt=7200010msec

Run status group 1 (all jobs):
   READ: io=4972.4GB, aggrb=724153KB/s, minb=724153KB/s, maxb=724153KB/s, mint=7200008msec, maxt=7200008msec

Run status group 2 (all jobs):
   READ: io=2330.6GB, aggrb=339411KB/s, minb=339411KB/s, maxb=339411KB/s, mint=7200008msec, maxt=7200008msec

Run status group 3 (all jobs):
   READ: io=2385.4GB, aggrb=347388KB/s, minb=347388KB/s, maxb=347388KB/s, mint=7200008msec, maxt=7200008msec



# 
# Thu 14 Jan 2016 05:11:21 AM EST
# 256kB write with SATA 1 jobs and 8 I/Os Outstanding
# 256kB write with NVMe 1 jobs and 1 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 1422 IOPS | 364 MBps | 5623 μsec Average Latency 
SATA-R5-1: 1395 IOPS | 357 MBps | 5731 μsec Average Latency 
NVMe-R5-0: 485 IOPS | 124 MBps | 2052 μsec Average Latency 
NVMe-R5-1: 481 IOPS | 123 MBps | 2069 μsec Average Latency 

Totals
IOPS: 3783
MBPS: 968
Average Latency (μsec): 3868

SATA-R5-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
NVMe-R5-11: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=106889: Thu Jan 14 07:12:22 2016
  write: io=2500.2GB, bw=364106KB/s, iops=1422, runt=7200005msec
    slat (usec): min=26, max=337, avg=53.42, stdev=14.75
    clat (usec): min=409, max=526710, avg=5569.51, stdev=15185.96
     lat (usec): min=515, max=526755, avg=5623.07, stdev=15185.60
    clat percentiles (usec):
     |  1.00th=[  748],  5.00th=[  988], 10.00th=[ 1192], 20.00th=[ 1464],
     | 30.00th=[ 1592], 40.00th=[ 1704], 50.00th=[ 1800], 60.00th=[ 1848],
     | 70.00th=[ 1960], 80.00th=[ 2192], 90.00th=[ 3984], 95.00th=[37632],
     | 99.00th=[69120], 99.50th=[96768], 99.90th=[201728], 99.95th=[203776],
     | 99.99th=[220160]
    bw (KB  /s): min=    4, max=768512, per=100.00%, avg=365463.68, stdev=89185.36
    lat (usec) : 500=0.01%, 750=1.05%, 1000=4.11%
    lat (msec) : 2=69.37%, 4=15.50%, 10=2.14%, 20=0.26%, 50=5.14%
    lat (msec) : 100=1.97%, 250=0.44%, 500=0.01%, 750=0.01%
  cpu          : usr=1.82%, sys=4.19%, ctx=19373876, majf=0, minf=67893
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10240477/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=106890: Thu Jan 14 07:12:22 2016
  write: io=2452.4GB, bw=357151KB/s, iops=1395, runt=7200039msec
    slat (usec): min=25, max=629, avg=81.98, stdev=26.68
    clat (usec): min=374, max=552020, avg=5649.38, stdev=15559.16
     lat (usec): min=568, max=552088, avg=5731.58, stdev=15558.79
    clat percentiles (usec):
     |  1.00th=[  756],  5.00th=[  988], 10.00th=[ 1176], 20.00th=[ 1432],
     | 30.00th=[ 1576], 40.00th=[ 1672], 50.00th=[ 1752], 60.00th=[ 1832],
     | 70.00th=[ 1928], 80.00th=[ 2160], 90.00th=[ 3856], 95.00th=[39680],
     | 99.00th=[70144], 99.50th=[95744], 99.90th=[201728], 99.95th=[203776],
     | 99.99th=[209920]
    bw (KB  /s): min=    4, max=722432, per=100.00%, avg=358496.53, stdev=88522.35
    lat (usec) : 500=0.01%, 750=0.92%, 1000=4.32%
    lat (msec) : 2=69.99%, 4=15.01%, 10=2.14%, 20=0.26%, 50=4.50%
    lat (msec) : 100=2.41%, 250=0.44%, 500=0.01%, 750=0.01%
  cpu          : usr=2.84%, sys=5.11%, ctx=17727627, majf=0, minf=66328
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=10044919/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=106891: Thu Jan 14 07:12:22 2016
  write: io=874714MB, bw=124403KB/s, iops=485, runt=7200041msec
    slat (usec): min=31, max=1966, avg=101.53, stdev=19.12
    clat (usec): min=258, max=606982, avg=1950.13, stdev=8838.97
     lat (usec): min=503, max=607084, avg=2052.07, stdev=8838.38
    clat percentiles (usec):
     |  1.00th=[  510],  5.00th=[  532], 10.00th=[  548], 20.00th=[  564],
     | 30.00th=[  580], 40.00th=[  596], 50.00th=[  620], 60.00th=[  652],
     | 70.00th=[  788], 80.00th=[ 1560], 90.00th=[ 1880], 95.00th=[ 2128],
     | 99.00th=[47872], 99.50th=[67072], 99.90th=[110080], 99.95th=[142336],
     | 99.99th=[203776]
    bw (KB  /s): min=    4, max=260096, per=100.00%, avg=124942.82, stdev=35018.29
    lat (usec) : 500=0.34%, 750=68.60%, 1000=4.30%
    lat (msec) : 2=20.51%, 4=4.09%, 10=0.37%, 20=0.07%, 50=0.79%
    lat (msec) : 100=0.79%, 250=0.15%, 500=0.01%, 750=0.01%
  cpu          : usr=1.12%, sys=2.18%, ctx=7048777, majf=0, minf=9309
  IO depths    : 1=100.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3498856/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=106892: Thu Jan 14 07:12:22 2016
  write: io=867129MB, bw=123324KB/s, iops=481, runt=7200038msec
    slat (usec): min=32, max=1231, avg=100.07, stdev=20.25
    clat (usec): min=2, max=325499, avg=1969.49, stdev=8928.61
     lat (usec): min=497, max=325601, avg=2069.95, stdev=8927.85
    clat percentiles (usec):
     |  1.00th=[  510],  5.00th=[  532], 10.00th=[  548], 20.00th=[  564],
     | 30.00th=[  580], 40.00th=[  596], 50.00th=[  612], 60.00th=[  652],
     | 70.00th=[  772], 80.00th=[ 1512], 90.00th=[ 1864], 95.00th=[ 2128],
     | 99.00th=[49408], 99.50th=[68096], 99.90th=[109056], 99.95th=[144384],
     | 99.99th=[203776]
    bw (KB  /s): min=    4, max=270336, per=100.00%, avg=123898.12, stdev=35482.47
    lat (usec) : 4=0.01%, 500=0.38%, 750=69.02%, 1000=4.27%
    lat (msec) : 2=20.16%, 4=3.97%, 10=0.36%, 20=0.06%, 50=0.79%
    lat (msec) : 100=0.84%, 250=0.14%, 500=0.01%
  cpu          : usr=1.17%, sys=2.15%, ctx=6989306, majf=0, minf=9435
  IO depths    : 1=100.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=3468517/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=2500.2GB, aggrb=364105KB/s, minb=364105KB/s, maxb=364105KB/s, mint=7200005msec, maxt=7200005msec

Run status group 1 (all jobs):
  WRITE: io=2452.4GB, aggrb=357150KB/s, minb=357150KB/s, maxb=357150KB/s, mint=7200039msec, maxt=7200039msec

Run status group 2 (all jobs):
  WRITE: io=874714MB, aggrb=124403KB/s, minb=124403KB/s, maxb=124403KB/s, mint=7200041msec, maxt=7200041msec

Run status group 3 (all jobs):
  WRITE: io=867129MB, aggrb=123324KB/s, minb=123324KB/s, maxb=123324KB/s, mint=7200038msec, maxt=7200038msec



# 
# Thu 14 Jan 2016 07:12:24 AM EST
# 256kB read with SATA 1 jobs and 8 I/Os Outstanding
# 256kB read with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 2285 IOPS | 584 MBps | 3498 μsec Average Latency 
SATA-R5-1: 2296 IOPS | 587 MBps | 3481 μsec Average Latency 
NVMe-R5-0: 1989 IOPS | 509 MBps | 2008 μsec Average Latency 
NVMe-R5-1: 1971 IOPS | 504 MBps | 2026 μsec Average Latency 

Totals
IOPS: 8541
MBPS: 2184
Average Latency (μsec): 2753

SATA-R5-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=112040: Thu Jan 14 09:13:25 2016
  read : io=4016.8GB, bw=584977KB/s, iops=2285, runt=7200003msec
    slat (usec): min=17, max=2026, avg=68.01, stdev=39.96
    clat (usec): min=511, max=25334, avg=3430.60, stdev=864.13
     lat (usec): min=974, max=25377, avg=3498.86, stdev=862.87
    clat percentiles (usec):
     |  1.00th=[ 2192],  5.00th=[ 2608], 10.00th=[ 2800], 20.00th=[ 2960],
     | 30.00th=[ 3088], 40.00th=[ 3152], 50.00th=[ 3280], 60.00th=[ 3376],
     | 70.00th=[ 3504], 80.00th=[ 3696], 90.00th=[ 4080], 95.00th=[ 4832],
     | 99.00th=[ 7200], 99.50th=[ 8160], 99.90th=[10688], 99.95th=[12096],
     | 99.99th=[15168]
    bw (KB  /s): min=    4, max=617472, per=100.00%, avg=585271.94, stdev=12965.65
    lat (usec) : 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.43%, 4=88.34%, 10=11.07%, 20=0.15%, 50=0.01%
  cpu          : usr=0.87%, sys=8.65%, ctx=31630567, majf=0, minf=68405
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=16452477/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=112041: Thu Jan 14 09:13:25 2016
  read : io=4036.6GB, bw=587863KB/s, iops=2296, runt=7200003msec
    slat (usec): min=17, max=1816, avg=67.57, stdev=37.25
    clat (usec): min=290, max=28811, avg=3413.74, stdev=886.38
     lat (usec): min=878, max=28871, avg=3481.55, stdev=885.23
    clat percentiles (usec):
     |  1.00th=[ 2128],  5.00th=[ 2576], 10.00th=[ 2736], 20.00th=[ 2928],
     | 30.00th=[ 3056], 40.00th=[ 3152], 50.00th=[ 3248], 60.00th=[ 3344],
     | 70.00th=[ 3504], 80.00th=[ 3664], 90.00th=[ 4128], 95.00th=[ 4896],
     | 99.00th=[ 7200], 99.50th=[ 8256], 99.90th=[10816], 99.95th=[12096],
     | 99.99th=[14912]
    bw (KB  /s): min=    4, max=632320, per=100.00%, avg=588161.55, stdev=12410.56
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.60%, 4=87.94%, 10=11.30%, 20=0.16%, 50=0.01%
  cpu          : usr=0.93%, sys=8.89%, ctx=31678756, majf=0, minf=66842
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=16533633/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=112042: Thu Jan 14 09:13:25 2016
  read : io=3497.5GB, bw=509355KB/s, iops=1989, runt=7200003msec
    slat (usec): min=18, max=2447, avg=46.98, stdev=24.12
    clat (usec): min=83, max=23228, avg=1961.32, stdev=642.24
     lat (usec): min=363, max=23262, avg=2008.46, stdev=641.63
    clat percentiles (usec):
     |  1.00th=[  940],  5.00th=[ 1272], 10.00th=[ 1432], 20.00th=[ 1592],
     | 30.00th=[ 1720], 40.00th=[ 1800], 50.00th=[ 1880], 60.00th=[ 1992],
     | 70.00th=[ 2096], 80.00th=[ 2224], 90.00th=[ 2448], 95.00th=[ 2672],
     | 99.00th=[ 4768], 99.50th=[ 5920], 99.90th=[ 8256], 99.95th=[ 9536],
     | 99.99th=[12352]
    bw (KB  /s): min=    4, max=568832, per=100.00%, avg=509645.87, stdev=12779.40
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.03%, 750=0.26%, 1000=1.04%
    lat (msec) : 2=59.90%, 4=37.40%, 10=1.33%, 20=0.04%, 50=0.01%
  cpu          : usr=0.86%, sys=7.09%, ctx=28109978, majf=0, minf=35035
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=14325599/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=112043: Thu Jan 14 09:13:25 2016
  read : io=3464.1GB, bw=504622KB/s, iops=1971, runt=7200002msec
    slat (usec): min=18, max=2967, avg=71.39, stdev=39.95
    clat (usec): min=1, max=23164, avg=1955.09, stdev=661.26
     lat (usec): min=384, max=23211, avg=2026.74, stdev=659.66
    clat percentiles (usec):
     |  1.00th=[  916],  5.00th=[ 1256], 10.00th=[ 1400], 20.00th=[ 1576],
     | 30.00th=[ 1688], 40.00th=[ 1784], 50.00th=[ 1880], 60.00th=[ 1976],
     | 70.00th=[ 2096], 80.00th=[ 2256], 90.00th=[ 2480], 95.00th=[ 2704],
     | 99.00th=[ 4832], 99.50th=[ 5984], 99.90th=[ 8384], 99.95th=[ 9664],
     | 99.99th=[12608]
    bw (KB  /s): min=    4, max=552960, per=100.00%, avg=504913.33, stdev=12241.54
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.04%
    lat (usec) : 750=0.31%, 1000=1.16%
    lat (msec) : 2=60.23%, 4=36.87%, 10=1.35%, 20=0.04%, 50=0.01%
  cpu          : usr=0.88%, sys=7.95%, ctx=27361175, majf=0, minf=34101
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=14192502/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=4016.8GB, aggrb=584976KB/s, minb=584976KB/s, maxb=584976KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=4036.6GB, aggrb=587862KB/s, minb=587862KB/s, maxb=587862KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=3497.5GB, aggrb=509354KB/s, minb=509354KB/s, maxb=509354KB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=3464.1GB, aggrb=504622KB/s, minb=504622KB/s, maxb=504622KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:13:28 AM EST
# 64kB write with SATA 1 jobs and 4 I/Os Outstanding
# 64kB write with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6041 IOPS | 386 MBps | 660 μsec Average Latency 
SATA-R5-1: 6130 IOPS | 392 MBps | 650 μsec Average Latency 
NVMe-R5-0: 2029 IOPS | 129 MBps | 1969 μsec Average Latency 
NVMe-R5-1: 1937 IOPS | 123 MBps | 2062 μsec Average Latency 

Totals
IOPS: 16137
MBPS: 1030
Average Latency (μsec): 1335

SATA-R5-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-01: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=117195: Thu Jan 14 11:14:28 2016
  write: io=2654.2GB, bw=386662KB/s, iops=6041, runt=7200001msec
    slat (usec): min=7, max=496, avg=26.03, stdev= 7.75
    clat (usec): min=128, max=420676, avg=634.63, stdev=3928.64
     lat (usec): min=195, max=420698, avg=660.86, stdev=3928.59
    clat percentiles (usec):
     |  1.00th=[  278],  5.00th=[  318], 10.00th=[  338], 20.00th=[  350],
     | 30.00th=[  370], 40.00th=[  390], 50.00th=[  430], 60.00th=[  454],
     | 70.00th=[  474], 80.00th=[  494], 90.00th=[  572], 95.00th=[  660],
     | 99.00th=[ 1400], 99.50th=[ 4128], 99.90th=[47360], 99.95th=[87552],
     | 99.99th=[195584]
    bw (KB  /s): min=    1, max=591232, per=100.00%, avg=388800.81, stdev=96906.94
    lat (usec) : 250=0.41%, 500=80.46%, 750=15.93%, 1000=1.59%
    lat (msec) : 2=0.92%, 4=0.20%, 10=0.09%, 20=0.05%, 50=0.26%
    lat (msec) : 100=0.06%, 250=0.04%, 500=0.01%
  cpu          : usr=2.52%, sys=7.14%, ctx=74637949, majf=0, minf=9472
  IO depths    : 1=0.1%, 2=0.1%, 4=101.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=43499485/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=117196: Thu Jan 14 11:14:28 2016
  write: io=2693.9GB, bw=392322KB/s, iops=6130, runt=7200001msec
    slat (usec): min=7, max=1858, avg=40.33, stdev=18.90
    clat (usec): min=92, max=466760, avg=610.27, stdev=3283.24
     lat (usec): min=210, max=466777, avg=650.82, stdev=3283.08
    clat percentiles (usec):
     |  1.00th=[  278],  5.00th=[  310], 10.00th=[  330], 20.00th=[  354],
     | 30.00th=[  386], 40.00th=[  418], 50.00th=[  438], 60.00th=[  458],
     | 70.00th=[  478], 80.00th=[  510], 90.00th=[  580], 95.00th=[  668],
     | 99.00th=[ 1368], 99.50th=[ 2800], 99.90th=[40704], 99.95th=[71168],
     | 99.99th=[154624]
    bw (KB  /s): min=    1, max=556928, per=100.00%, avg=393201.35, stdev=66999.77
    lat (usec) : 100=0.01%, 250=0.26%, 500=77.23%, 750=19.40%, 1000=1.52%
    lat (msec) : 2=0.94%, 4=0.22%, 10=0.09%, 20=0.03%, 50=0.23%
    lat (msec) : 100=0.05%, 250=0.02%, 500=0.01%
  cpu          : usr=3.33%, sys=11.54%, ctx=73296292, majf=0, minf=9401
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=44136195/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=117197: Thu Jan 14 11:14:28 2016
  write: io=913482MB, bw=129915KB/s, iops=2029, runt=7200157msec
    slat (usec): min=7, max=242, avg=25.37, stdev=11.75
    clat (usec): min=95, max=501020, avg=1943.67, stdev=12820.70
     lat (usec): min=198, max=501051, avg=1969.22, stdev=12820.65
    clat percentiles (usec):
     |  1.00th=[  286],  5.00th=[  330], 10.00th=[  346], 20.00th=[  382],
     | 30.00th=[  426], 40.00th=[  454], 50.00th=[  474], 60.00th=[  490],
     | 70.00th=[  524], 80.00th=[  588], 90.00th=[  684], 95.00th=[  876],
     | 99.00th=[51456], 99.50th=[93696], 99.90th=[199680], 99.95th=[199680],
     | 99.99th=[268288]
    bw (KB  /s): min=    1, max=391552, per=100.00%, avg=131148.29, stdev=50384.69
    lat (usec) : 100=0.01%, 250=0.25%, 500=63.01%, 750=29.44%, 1000=3.38%
    lat (msec) : 2=1.56%, 4=0.32%, 10=0.15%, 20=0.06%, 50=0.80%
    lat (msec) : 100=0.58%, 250=0.44%, 500=0.02%, 750=0.01%
  cpu          : usr=0.96%, sys=2.58%, ctx=24982608, majf=0, minf=9457
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=14615705/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=117198: Thu Jan 14 11:14:28 2016
  write: io=871814MB, bw=123989KB/s, iops=1937, runt=7200126msec
    slat (usec): min=8, max=475, avg=38.52, stdev=23.69
    clat (usec): min=70, max=489060, avg=2024.21, stdev=13284.88
     lat (usec): min=223, max=489087, avg=2062.92, stdev=13284.76
    clat percentiles (usec):
     |  1.00th=[  278],  5.00th=[  330], 10.00th=[  358], 20.00th=[  402],
     | 30.00th=[  438], 40.00th=[  462], 50.00th=[  482], 60.00th=[  510],
     | 70.00th=[  548], 80.00th=[  596], 90.00th=[  684], 95.00th=[  852],
     | 99.00th=[53504], 99.50th=[99840], 99.90th=[199680], 99.95th=[201728],
     | 99.99th=[272384]
    bw (KB  /s): min=    1, max=368256, per=100.00%, avg=125140.93, stdev=47872.02
    lat (usec) : 100=0.01%, 250=0.35%, 500=56.40%, 750=36.30%, 1000=3.07%
    lat (msec) : 2=1.49%, 4=0.33%, 10=0.14%, 20=0.05%, 50=0.77%
    lat (msec) : 100=0.61%, 250=0.48%, 500=0.02%
  cpu          : usr=1.09%, sys=3.44%, ctx=23005150, majf=0, minf=9511
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13949021/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=2654.2GB, aggrb=386662KB/s, minb=386662KB/s, maxb=386662KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=2693.9GB, aggrb=392321KB/s, minb=392321KB/s, maxb=392321KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=913482MB, aggrb=129914KB/s, minb=129914KB/s, maxb=129914KB/s, mint=7200157msec, maxt=7200157msec

Run status group 3 (all jobs):
  WRITE: io=871814MB, aggrb=123989KB/s, minb=123989KB/s, maxb=123989KB/s, mint=7200126msec, maxt=7200126msec



# 
# Thu 14 Jan 2016 11:14:30 AM EST
# 64kB read with SATA 1 jobs and 10 I/Os Outstanding
# 64kB read with NVMe 1 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 7189 IOPS | 460 MBps | 1389 μsec Average Latency 
SATA-R5-1: 7242 IOPS | 463 MBps | 1379 μsec Average Latency 
NVMe-R5-0: 9196 IOPS | 588 MBps | 868 μsec Average Latency 
NVMe-R5-1: 9186 IOPS | 587 MBps | 869 μsec Average Latency 

Totals
IOPS: 32813
MBPS: 2098
Average Latency (μsec): 1126

SATA-R5-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
SATA-R5-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
NVMe-R5-01: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
NVMe-R5-11: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 4 processes

SATA-R5-01: (groupid=0, jobs=1): err= 0: pid=122307: Thu Jan 14 13:15:31 2016
  read : io=3159.5GB, bw=460129KB/s, iops=7189, runt=7200001msec
    slat (usec): min=5, max=3033, avg=24.40, stdev=23.14
    clat (usec): min=1, max=43966, avg=1365.22, stdev=486.27
     lat (usec): min=166, max=43981, avg=1389.79, stdev=485.17
    clat percentiles (usec):
     |  1.00th=[  580],  5.00th=[  812], 10.00th=[  932], 20.00th=[ 1064],
     | 30.00th=[ 1144], 40.00th=[ 1240], 50.00th=[ 1304], 60.00th=[ 1384],
     | 70.00th=[ 1496], 80.00th=[ 1608], 90.00th=[ 1800], 95.00th=[ 2008],
     | 99.00th=[ 3024], 99.50th=[ 3632], 99.90th=[ 6112], 99.95th=[ 8096],
     | 99.99th=[10944]
    bw (KB  /s): min=    1, max=505216, per=100.00%, avg=460253.31, stdev=13816.20
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.49%
    lat (usec) : 750=2.89%, 1000=11.49%
    lat (msec) : 2=79.99%, 4=4.80%, 10=0.32%, 20=0.02%, 50=0.01%
  cpu          : usr=1.37%, sys=9.38%, ctx=77620963, majf=0, minf=23494
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=51764526/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
SATA-R5-11: (groupid=1, jobs=1): err= 0: pid=122308: Thu Jan 14 13:15:31 2016
  read : io=3182.6GB, bw=463490KB/s, iops=7242, runt=7200001msec
    slat (usec): min=6, max=1987, avg=16.22, stdev=14.32
    clat (usec): min=1, max=20549, avg=1363.53, stdev=532.07
     lat (usec): min=183, max=20562, avg=1379.87, stdev=531.69
    clat percentiles (usec):
     |  1.00th=[  548],  5.00th=[  780], 10.00th=[  900], 20.00th=[ 1032],
     | 30.00th=[ 1128], 40.00th=[ 1224], 50.00th=[ 1288], 60.00th=[ 1368],
     | 70.00th=[ 1480], 80.00th=[ 1608], 90.00th=[ 1816], 95.00th=[ 2064],
     | 99.00th=[ 3408], 99.50th=[ 3952], 99.90th=[ 6368], 99.95th=[ 8512],
     | 99.99th=[11072]
    bw (KB  /s): min=    1, max=509312, per=100.00%, avg=463619.54, stdev=14523.97
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.65%, 750=3.42%, 1000=12.67%
    lat (msec) : 2=77.32%, 4=5.45%, 10=0.45%, 20=0.03%, 50=0.01%
  cpu          : usr=1.25%, sys=7.74%, ctx=81152852, majf=0, minf=22136
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=52142639/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-01: (groupid=2, jobs=1): err= 0: pid=122309: Thu Jan 14 13:15:31 2016
  read : io=4041.7GB, bw=588599KB/s, iops=9196, runt=7200001msec
    slat (usec): min=6, max=1495, avg=25.01, stdev=23.20
    clat (usec): min=1, max=15529, avg=843.60, stdev=361.78
     lat (usec): min=182, max=15542, avg=868.77, stdev=360.15
    clat percentiles (usec):
     |  1.00th=[  366],  5.00th=[  470], 10.00th=[  516], 20.00th=[  604],
     | 30.00th=[  668], 40.00th=[  724], 50.00th=[  780], 60.00th=[  852],
     | 70.00th=[  932], 80.00th=[ 1048], 90.00th=[ 1224], 95.00th=[ 1384],
     | 99.00th=[ 1800], 99.50th=[ 2064], 99.90th=[ 4384], 99.95th=[ 6304],
     | 99.99th=[10176]
    bw (KB  /s): min=    1, max=635520, per=100.00%, avg=588713.37, stdev=17532.72
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.07%, 500=7.74%, 750=36.68%, 1000=31.91%
    lat (msec) : 2=23.03%, 4=0.45%, 10=0.10%, 20=0.01%
  cpu          : usr=1.66%, sys=12.15%, ctx=95972454, majf=0, minf=17872
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=66217370/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=1): err= 0: pid=122310: Thu Jan 14 13:15:31 2016
  read : io=4036.9GB, bw=587907KB/s, iops=9186, runt=7200001msec
    slat (usec): min=6, max=1954, avg=18.59, stdev=17.94
    clat (usec): min=0, max=19611, avg=851.18, stdev=374.87
     lat (usec): min=173, max=19639, avg=869.88, stdev=374.10
    clat percentiles (usec):
     |  1.00th=[  358],  5.00th=[  466], 10.00th=[  510], 20.00th=[  604],
     | 30.00th=[  660], 40.00th=[  724], 50.00th=[  780], 60.00th=[  860],
     | 70.00th=[  956], 80.00th=[ 1064], 90.00th=[ 1256], 95.00th=[ 1416],
     | 99.00th=[ 1832], 99.50th=[ 2096], 99.90th=[ 4448], 99.95th=[ 6432],
     | 99.99th=[10304]
    bw (KB  /s): min=    1, max=641792, per=100.00%, avg=588014.37, stdev=19729.39
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.07%, 500=8.68%, 750=36.46%, 1000=29.76%
    lat (msec) : 2=24.42%, 4=0.50%, 10=0.10%, 20=0.01%
  cpu          : usr=1.56%, sys=10.13%, ctx=99890960, majf=0, minf=19664
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=66139485/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3159.5GB, aggrb=460129KB/s, minb=460129KB/s, maxb=460129KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=3182.6GB, aggrb=463490KB/s, minb=463490KB/s, maxb=463490KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=4041.7GB, aggrb=588598KB/s, minb=588598KB/s, maxb=588598KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=4036.9GB, aggrb=587906KB/s, minb=587906KB/s, maxb=587906KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 01:15:34 PM EST
# 64kB randread with SATA 2 jobs and 2 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 3556 IOPS | 227 MBps | 1121 μsec Average Latency 
SATA-R5-1: 3735 IOPS | 239 MBps | 1068 μsec Average Latency 
NVMe-R5-0: 8671 IOPS | 554 MBps | 920 μsec Average Latency 
NVMe-R5-1: 7425 IOPS | 475 MBps | 1074 μsec Average Latency 

Totals
IOPS: 23387
MBPS: 1495
Average Latency (μsec): 1045

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 12 processes

SATA-R5-01: (groupid=0, jobs=2): err= 0: pid=127462: Thu Jan 14 15:16:34 2016
  read : io=1562.9GB, bw=227601KB/s, iops=3556, runt=7200001msec
    slat (usec): min=6, max=1004, avg=33.56, stdev=30.12
    clat (usec): min=54, max=20302, avg=1088.10, stdev=347.72
     lat (usec): min=205, max=20321, avg=1121.92, stdev=345.71
    clat percentiles (usec):
     |  1.00th=[  724],  5.00th=[  812], 10.00th=[  860], 20.00th=[  916],
     | 30.00th=[  964], 40.00th=[ 1012], 50.00th=[ 1048], 60.00th=[ 1096],
     | 70.00th=[ 1144], 80.00th=[ 1208], 90.00th=[ 1304], 95.00th=[ 1400],
     | 99.00th=[ 1848], 99.50th=[ 3024], 99.90th=[ 5728], 99.95th=[ 6816],
     | 99.99th=[11200]
    bw (KB  /s): min=    1, max=122112, per=50.03%, avg=113873.88, stdev=2867.18
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.03%, 750=1.71%, 1000=36.13%
    lat (msec) : 2=61.25%, 4=0.66%, 10=0.21%, 20=0.02%, 50=0.01%
  cpu          : usr=0.82%, sys=3.56%, ctx=49360814, majf=0, minf=11853
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=25605070/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=2): err= 0: pid=127464: Thu Jan 14 15:16:34 2016
  read : io=1641.4GB, bw=239044KB/s, iops=3735, runt=7200001msec
    slat (usec): min=6, max=1026, avg=30.72, stdev=26.99
    clat (usec): min=1, max=22540, avg=1037.29, stdev=347.29
     lat (usec): min=177, max=22557, avg=1068.22, stdev=345.87
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[  724], 10.00th=[  788], 20.00th=[  860],
     | 30.00th=[  916], 40.00th=[  964], 50.00th=[ 1004], 60.00th=[ 1048],
     | 70.00th=[ 1112], 80.00th=[ 1176], 90.00th=[ 1272], 95.00th=[ 1368],
     | 99.00th=[ 1704], 99.50th=[ 2672], 99.90th=[ 5728], 99.95th=[ 6880],
     | 99.99th=[11456]
    bw (KB  /s): min=    1, max=130432, per=50.03%, avg=119596.91, stdev=3372.96
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.08%
    lat (usec) : 750=6.67%, 1000=41.50%
    lat (msec) : 2=51.05%, 4=0.49%, 10=0.18%, 20=0.02%, 50=0.01%
  cpu          : usr=0.83%, sys=3.58%, ctx=51982473, majf=0, minf=11257
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26892452/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=127466: Thu Jan 14 15:16:34 2016
  read : io=3810.8GB, bw=554978KB/s, iops=8671, runt=7200001msec
    slat (usec): min=6, max=2419, avg=33.34, stdev=29.29
    clat (usec): min=0, max=19761, avg=886.47, stdev=366.49
     lat (usec): min=173, max=19842, avg=920.03, stdev=364.65
    clat percentiles (usec):
     |  1.00th=[  330],  5.00th=[  490], 10.00th=[  596], 20.00th=[  684],
     | 30.00th=[  740], 40.00th=[  788], 50.00th=[  844], 60.00th=[  900],
     | 70.00th=[  964], 80.00th=[ 1048], 90.00th=[ 1208], 95.00th=[ 1368],
     | 99.00th=[ 1816], 99.50th=[ 2128], 99.90th=[ 5408], 99.95th=[ 6624],
     | 99.99th=[10816]
    bw (KB  /s): min=    1, max=174720, per=25.01%, avg=138817.99, stdev=6355.28
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.20%, 500=5.11%, 750=26.75%, 1000=42.51%
    lat (msec) : 2=24.79%, 4=0.47%, 10=0.15%, 20=0.02%
  cpu          : usr=0.90%, sys=4.35%, ctx=119461062, majf=0, minf=22991
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=62434983/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=127470: Thu Jan 14 15:16:34 2016
  read : io=3263.2GB, bw=475228KB/s, iops=7425, runt=7200001msec
    slat (usec): min=5, max=2472, avg=34.09, stdev=29.05
    clat (usec): min=0, max=24135, avg=1040.33, stdev=656.15
     lat (usec): min=172, max=24154, avg=1074.65, stdev=655.28
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  486], 10.00th=[  588], 20.00th=[  676],
     | 30.00th=[  740], 40.00th=[  796], 50.00th=[  860], 60.00th=[  932],
     | 70.00th=[ 1032], 80.00th=[ 1224], 90.00th=[ 1736], 95.00th=[ 2288],
     | 99.00th=[ 3664], 99.50th=[ 4320], 99.90th=[ 6176], 99.95th=[ 7264],
     | 99.99th=[11584]
    bw (KB  /s): min=    1, max=149120, per=25.02%, avg=118879.42, stdev=12219.15
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.24%, 500=5.20%, 750=26.61%, 1000=35.30%
    lat (msec) : 2=25.39%, 4=6.55%, 10=0.68%, 20=0.02%, 50=0.01%
  cpu          : usr=0.82%, sys=3.85%, ctx=103202600, majf=0, minf=23356
  IO depths    : 1=0.1%, 2=101.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=53463166/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1562.9GB, aggrb=227600KB/s, minb=227600KB/s, maxb=227600KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1641.4GB, aggrb=239044KB/s, minb=239044KB/s, maxb=239044KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=3810.8GB, aggrb=554977KB/s, minb=554977KB/s, maxb=554977KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=3263.2GB, aggrb=475228KB/s, minb=475228KB/s, maxb=475228KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 03:16:37 PM EST
# 64kB randread with SATA 6 jobs and 4 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 6288 IOPS | 402 MBps | 3814 μsec Average Latency 
SATA-R5-1: 7380 IOPS | 472 MBps | 3249 μsec Average Latency 
NVMe-R5-0: 9128 IOPS | 584 MBps | 3503 μsec Average Latency 
NVMe-R5-1: 7712 IOPS | 493 MBps | 4146 μsec Average Latency 

Totals
IOPS: 30508
MBPS: 1951
Average Latency (μsec): 3678

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=132607: Thu Jan 14 17:17:38 2016
  read : io=2763.5GB, bw=402448KB/s, iops=6288, runt=7200004msec
    slat (usec): min=6, max=3646, avg=30.60, stdev=38.37
    clat (usec): min=0, max=24566, avg=3783.28, stdev=1490.46
     lat (usec): min=305, max=24579, avg=3814.12, stdev=1489.40
    clat percentiles (usec):
     |  1.00th=[ 1736],  5.00th=[ 2448], 10.00th=[ 2736], 20.00th=[ 2960],
     | 30.00th=[ 3120], 40.00th=[ 3216], 50.00th=[ 3312], 60.00th=[ 3440],
     | 70.00th=[ 3664], 80.00th=[ 4192], 90.00th=[ 5856], 95.00th=[ 7136],
     | 99.00th=[ 9536], 99.50th=[10560], 99.90th=[12736], 99.95th=[13632],
     | 99.99th=[15680]
    bw (KB  /s): min=    1, max=74112, per=16.68%, avg=67125.38, stdev=1645.70
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.04%
    lat (msec) : 2=1.79%, 4=76.09%, 10=21.38%, 20=0.70%, 50=0.01%
  cpu          : usr=0.44%, sys=2.00%, ctx=88071777, majf=0, minf=70339
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=45275451/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=132613: Thu Jan 14 17:17:38 2016
  read : io=3243.6GB, bw=472380KB/s, iops=7380, runt=7200002msec
    slat (usec): min=6, max=3797, avg=33.03, stdev=37.37
    clat (usec): min=1, max=25322, avg=3215.84, stdev=1535.36
     lat (usec): min=257, max=25338, avg=3249.12, stdev=1534.46
    clat percentiles (usec):
     |  1.00th=[ 1336],  5.00th=[ 1768], 10.00th=[ 2008], 20.00th=[ 2288],
     | 30.00th=[ 2480], 40.00th=[ 2640], 50.00th=[ 2800], 60.00th=[ 2960],
     | 70.00th=[ 3216], 80.00th=[ 3632], 90.00th=[ 5280], 95.00th=[ 6688],
     | 99.00th=[ 9024], 99.50th=[10048], 99.90th=[12352], 99.95th=[13248],
     | 99.99th=[15168]
    bw (KB  /s): min=    1, max=102144, per=16.68%, avg=78787.87, stdev=5955.89
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.13%
    lat (msec) : 2=9.76%, 4=74.04%, 10=15.53%, 20=0.53%, 50=0.01%
  cpu          : usr=0.50%, sys=2.31%, ctx=101966689, majf=0, minf=67949
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=53142787/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=132619: Thu Jan 14 17:17:38 2016
  read : io=4011.4GB, bw=584194KB/s, iops=9128, runt=7200003msec
    slat (usec): min=5, max=3714, avg=31.82, stdev=36.93
    clat (usec): min=1, max=27362, avg=3471.39, stdev=1475.05
     lat (usec): min=249, max=27482, avg=3503.45, stdev=1474.09
    clat percentiles (usec):
     |  1.00th=[ 1256],  5.00th=[ 1992], 10.00th=[ 2320], 20.00th=[ 2640],
     | 30.00th=[ 2800], 40.00th=[ 2928], 50.00th=[ 3056], 60.00th=[ 3184],
     | 70.00th=[ 3440], 80.00th=[ 3952], 90.00th=[ 5408], 95.00th=[ 6688],
     | 99.00th=[ 9024], 99.50th=[10048], 99.90th=[12480], 99.95th=[13376],
     | 99.99th=[15424]
    bw (KB  /s): min=    1, max=163072, per=25.01%, avg=146132.87, stdev=5470.03
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.02%, 750=0.10%, 1000=0.31%
    lat (msec) : 2=4.72%, 4=75.24%, 10=19.06%, 20=0.54%, 50=0.01%
  cpu          : usr=0.81%, sys=4.05%, ctx=120231336, majf=0, minf=87119
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=65721871/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=132623: Thu Jan 14 17:17:38 2016
  read : io=3389.3GB, bw=493599KB/s, iops=7712, runt=7200004msec
    slat (usec): min=6, max=3517, avg=32.45, stdev=36.40
    clat (usec): min=1, max=74963, avg=4114.05, stdev=3391.88
     lat (usec): min=205, max=74978, avg=4146.76, stdev=3391.49
    clat percentiles (usec):
     |  1.00th=[  852],  5.00th=[ 1320], 10.00th=[ 1624], 20.00th=[ 1944],
     | 30.00th=[ 2160], 40.00th=[ 2384], 50.00th=[ 2608], 60.00th=[ 3024],
     | 70.00th=[ 4448], 80.00th=[ 6432], 90.00th=[ 8512], 95.00th=[10560],
     | 99.00th=[17280], 99.50th=[19328], 99.90th=[24704], 99.95th=[27520],
     | 99.99th=[34048]
    bw (KB  /s): min=    1, max=159104, per=25.02%, avg=123477.19, stdev=13541.71
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.11%, 750=0.46%, 1000=1.31%
    lat (msec) : 2=20.26%, 4=45.84%, 10=25.97%, 20=5.63%, 50=0.41%
    lat (msec) : 100=0.01%
  cpu          : usr=0.73%, sys=3.47%, ctx=102822056, majf=0, minf=83102
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=101.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=55529931/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=2763.5GB, aggrb=402448KB/s, minb=402448KB/s, maxb=402448KB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=3243.6GB, aggrb=472380KB/s, minb=472380KB/s, maxb=472380KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=4011.4GB, aggrb=584194KB/s, minb=584194KB/s, maxb=584194KB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=3389.3GB, aggrb=493599KB/s, minb=493599KB/s, maxb=493599KB/s, mint=7200004msec, maxt=7200004msec



# 
# Thu 14 Jan 2016 05:17:40 PM EST
# 32kB randread with SATA 5 jobs and 2 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 9495 IOPS | 303 MBps | 1050 μsec Average Latency 
SATA-R5-1: 9841 IOPS | 314 MBps | 1014 μsec Average Latency 
NVMe-R5-0: 13689 IOPS | 438 MBps | 874 μsec Average Latency 
NVMe-R5-1: 13105 IOPS | 419 MBps | 914 μsec Average Latency 

Totals
IOPS: 46130
MBPS: 1474
Average Latency (μsec): 963

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=137788: Thu Jan 14 19:18:41 2016
  read : io=2086.4GB, bw=303847KB/s, iops=9495, runt=7200002msec
    slat (usec): min=3, max=2477, avg=26.02, stdev=28.28
    clat (usec): min=1, max=26991, avg=1024.55, stdev=552.58
     lat (usec): min=186, max=27008, avg=1050.80, stdev=551.37
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  692], 10.00th=[  740], 20.00th=[  812],
     | 30.00th=[  860], 40.00th=[  908], 50.00th=[  948], 60.00th=[  996],
     | 70.00th=[ 1048], 80.00th=[ 1128], 90.00th=[ 1256], 95.00th=[ 1400],
     | 99.00th=[ 3248], 99.50th=[ 4960], 99.90th=[ 8640], 99.95th=[10432],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=67648, per=20.01%, avg=60807.22, stdev=2414.05
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.21%, 750=10.64%, 1000=49.63%
    lat (msec) : 2=38.00%, 4=0.69%, 10=0.76%, 20=0.06%, 50=0.01%
  cpu          : usr=0.74%, sys=2.90%, ctx=130771140, majf=0, minf=20217
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=68365524/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=137793: Thu Jan 14 19:18:41 2016
  read : io=2162.5GB, bw=314925KB/s, iops=9841, runt=7200001msec
    slat (usec): min=3, max=2962, avg=22.50, stdev=26.09
    clat (usec): min=0, max=23851, avg=991.33, stdev=554.72
     lat (usec): min=162, max=23865, avg=1014.02, stdev=553.90
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  660], 10.00th=[  716], 20.00th=[  772],
     | 30.00th=[  828], 40.00th=[  868], 50.00th=[  916], 60.00th=[  964],
     | 70.00th=[ 1020], 80.00th=[ 1096], 90.00th=[ 1224], 95.00th=[ 1368],
     | 99.00th=[ 3184], 99.50th=[ 4896], 99.90th=[ 8640], 99.95th=[10560],
     | 99.99th=[12608]
    bw (KB  /s): min=    0, max=71296, per=20.01%, avg=63022.57, stdev=2342.99
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.31%, 750=15.71%, 1000=50.82%
    lat (msec) : 2=31.67%, 4=0.69%, 10=0.74%, 20=0.06%, 50=0.01%
  cpu          : usr=0.66%, sys=2.61%, ctx=136450703, majf=0, minf=19408
  IO depths    : 1=0.1%, 2=100.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=70858087/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=137798: Thu Jan 14 19:18:41 2016
  read : io=3007.1GB, bw=438066KB/s, iops=13689, runt=7200001msec
    slat (usec): min=3, max=2089, avg=23.73, stdev=26.34
    clat (usec): min=0, max=22198, avg=850.82, stdev=531.18
     lat (usec): min=136, max=22207, avg=874.76, stdev=529.82
    clat percentiles (usec):
     |  1.00th=[  378],  5.00th=[  502], 10.00th=[  564], 20.00th=[  636],
     | 30.00th=[  692], 40.00th=[  732], 50.00th=[  772], 60.00th=[  820],
     | 70.00th=[  884], 80.00th=[  964], 90.00th=[ 1112], 95.00th=[ 1272],
     | 99.00th=[ 2352], 99.50th=[ 4640], 99.90th=[ 8096], 99.95th=[10048],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=166656, per=33.34%, avg=146069.32, stdev=7447.03
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.08%, 500=4.88%, 750=39.21%, 1000=39.06%
    lat (msec) : 2=15.57%, 4=0.51%, 10=0.63%, 20=0.05%, 50=0.01%
  cpu          : usr=1.29%, sys=5.88%, ctx=169949800, majf=0, minf=20168
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=98564933/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=137801: Thu Jan 14 19:18:41 2016
  read : io=2879.7GB, bw=419376KB/s, iops=13105, runt=7200001msec
    slat (usec): min=3, max=1711, avg=16.28, stdev=20.25
    clat (usec): min=0, max=25425, avg=897.61, stdev=620.22
     lat (usec): min=141, max=25432, avg=914.04, stdev=619.69
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  498], 10.00th=[  564], 20.00th=[  628],
     | 30.00th=[  684], 40.00th=[  732], 50.00th=[  772], 60.00th=[  828],
     | 70.00th=[  900], 80.00th=[ 1004], 90.00th=[ 1224], 95.00th=[ 1544],
     | 99.00th=[ 3728], 99.50th=[ 4896], 99.90th=[ 8512], 99.95th=[10304],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=164352, per=33.35%, avg=139842.81, stdev=11563.90
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.08%, 500=5.06%, 750=39.61%, 1000=34.89%
    lat (msec) : 2=17.62%, 4=1.87%, 10=0.82%, 20=0.06%, 50=0.01%
  cpu          : usr=1.08%, sys=4.33%, ctx=167262943, majf=0, minf=17873
  IO depths    : 1=0.1%, 2=0.1%, 4=100.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=94359525/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=2086.4GB, aggrb=303846KB/s, minb=303846KB/s, maxb=303846KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=2162.5GB, aggrb=314924KB/s, minb=314924KB/s, maxb=314924KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=3007.1GB, aggrb=438066KB/s, minb=438066KB/s, maxb=438066KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=2879.7GB, aggrb=419375KB/s, minb=419375KB/s, maxb=419375KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 07:18:44 PM EST
# 32kB randread with SATA 4 jobs and 16 I/Os Outstanding
# 32kB randread with NVMe 3 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 17883 IOPS | 572 MBps | 3577 μsec Average Latency 
SATA-R5-1: 18011 IOPS | 576 MBps | 3551 μsec Average Latency 
NVMe-R5-0: 9728 IOPS | 311 MBps | 2465 μsec Average Latency 
NVMe-R5-1: 9671 IOPS | 309 MBps | 2479 μsec Average Latency 

Totals
IOPS: 55293
MBPS: 1768
Average Latency (μsec): 3018

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 14 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=142963: Thu Jan 14 21:19:44 2016
  read : io=3929.5GB, bw=572269KB/s, iops=17883, runt=7200003msec
    slat (usec): min=3, max=5917, avg=16.96, stdev=35.63
    clat (usec): min=0, max=102022, avg=3559.98, stdev=1575.65
     lat (usec): min=217, max=102040, avg=3577.11, stdev=1575.16
    clat percentiles (usec):
     |  1.00th=[ 1544],  5.00th=[ 2064], 10.00th=[ 2256], 20.00th=[ 2512],
     | 30.00th=[ 2736], 40.00th=[ 2928], 50.00th=[ 3152], 60.00th=[ 3376],
     | 70.00th=[ 3696], 80.00th=[ 4192], 90.00th=[ 5408], 95.00th=[ 7136],
     | 99.00th=[ 9536], 99.50th=[10560], 99.90th=[12864], 99.95th=[13760],
     | 99.99th=[16064]
    bw (KB  /s): min=    0, max=171200, per=25.01%, avg=143127.13, stdev=5101.32
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.06%
    lat (msec) : 2=4.08%, 4=72.88%, 10=22.25%, 20=0.72%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.07%, sys=4.49%, ctx=219712325, majf=0, minf=74412
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=128760516/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=142967: Thu Jan 14 21:19:44 2016
  read : io=3957.7GB, bw=576371KB/s, iops=18011, runt=7200003msec
    slat (usec): min=4, max=5317, avg=24.98, stdev=34.39
    clat (usec): min=1, max=128794, avg=3526.36, stdev=1572.93
     lat (usec): min=271, max=128822, avg=3551.54, stdev=1572.30
    clat percentiles (usec):
     |  1.00th=[ 1544],  5.00th=[ 2024], 10.00th=[ 2224], 20.00th=[ 2480],
     | 30.00th=[ 2704], 40.00th=[ 2896], 50.00th=[ 3120], 60.00th=[ 3344],
     | 70.00th=[ 3664], 80.00th=[ 4128], 90.00th=[ 5344], 95.00th=[ 7072],
     | 99.00th=[ 9408], 99.50th=[10560], 99.90th=[12864], 99.95th=[13760],
     | 99.99th=[15936]
    bw (KB  /s): min=    0, max=169280, per=25.01%, avg=144153.15, stdev=5105.49
    lat (usec) : 2=0.01%, 20=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.01%, 1000=0.07%
    lat (msec) : 2=4.46%, 4=73.30%, 10=21.46%, 20=0.71%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.22%, sys=5.67%, ctx=211908848, majf=0, minf=76254
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.9%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=129683420/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=3): err= 0: pid=142971: Thu Jan 14 21:19:44 2016
  read : io=2137.7GB, bw=311318KB/s, iops=9728, runt=7200002msec
    slat (usec): min=3, max=6434, avg=16.43, stdev=34.96
    clat (usec): min=0, max=98356, avg=2448.72, stdev=1416.04
     lat (usec): min=158, max=98366, avg=2465.32, stdev=1415.79
    clat percentiles (usec):
     |  1.00th=[  836],  5.00th=[ 1272], 10.00th=[ 1464], 20.00th=[ 1640],
     | 30.00th=[ 1768], 40.00th=[ 1880], 50.00th=[ 2024], 60.00th=[ 2192],
     | 70.00th=[ 2448], 80.00th=[ 2896], 90.00th=[ 3728], 95.00th=[ 5728],
     | 99.00th=[ 8256], 99.50th=[ 9280], 99.90th=[11584], 99.95th=[12480],
     | 99.99th=[14528]
    bw (KB  /s): min=    0, max=123648, per=33.35%, avg=103824.80, stdev=4374.24
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.04%, 750=0.54%, 1000=1.45%
    lat (msec) : 2=46.62%, 4=43.01%, 10=8.03%, 20=0.31%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.81%, sys=3.24%, ctx=124859337, majf=0, minf=29943
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=70046524/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=3): err= 0: pid=142974: Thu Jan 14 21:19:44 2016
  read : io=2125.6GB, bw=309484KB/s, iops=9671, runt=7200002msec
    slat (usec): min=4, max=3677, avg=25.65, stdev=36.08
    clat (usec): min=2, max=132079, avg=2453.77, stdev=1417.28
     lat (usec): min=204, max=132088, avg=2479.62, stdev=1416.82
    clat percentiles (usec):
     |  1.00th=[  876],  5.00th=[ 1272], 10.00th=[ 1448], 20.00th=[ 1640],
     | 30.00th=[ 1768], 40.00th=[ 1896], 50.00th=[ 2024], 60.00th=[ 2192],
     | 70.00th=[ 2480], 80.00th=[ 2896], 90.00th=[ 3728], 95.00th=[ 5728],
     | 99.00th=[ 8256], 99.50th=[ 9280], 99.90th=[11584], 99.95th=[12480],
     | 99.99th=[14528]
    bw (KB  /s): min=    0, max=120832, per=33.35%, avg=103213.97, stdev=4470.03
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.03%, 750=0.42%, 1000=1.33%
    lat (msec) : 2=46.07%, 4=43.87%, 10=7.97%, 20=0.31%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.01%, sys=4.25%, ctx=119744379, majf=0, minf=29448
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=69633844/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3929.5GB, aggrb=572268KB/s, minb=572268KB/s, maxb=572268KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=3957.7GB, aggrb=576370KB/s, minb=576370KB/s, maxb=576370KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=2137.7GB, aggrb=311317KB/s, minb=311317KB/s, maxb=311317KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=2125.6GB, aggrb=309483KB/s, minb=309483KB/s, maxb=309483KB/s, mint=7200002msec, maxt=7200002msec



# 
# Thu 14 Jan 2016 09:19:47 PM EST
# 16kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 16kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 28139 IOPS | 450 MBps | 2841 μsec Average Latency 
SATA-R5-1: 28087 IOPS | 449 MBps | 2846 μsec Average Latency 
NVMe-R5-0: 6483 IOPS | 103 MBps | 2465 μsec Average Latency 
NVMe-R5-1: 6686 IOPS | 106 MBps | 2390 μsec Average Latency 

Totals
IOPS: 69395
MBPS: 1108
Average Latency (μsec): 2635

SATA-R5-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=148100: Thu Jan 14 23:20:48 2016
  read : io=3091.6GB, bw=450237KB/s, iops=28139, runt=7200003msec
    slat (usec): min=2, max=3407, avg=19.41, stdev=22.37
    clat (usec): min=79, max=162595, avg=2821.71, stdev=598.59
     lat (usec): min=226, max=162607, avg=2841.31, stdev=597.43
    clat percentiles (usec):
     |  1.00th=[ 1864],  5.00th=[ 2160], 10.00th=[ 2288], 20.00th=[ 2448],
     | 30.00th=[ 2544], 40.00th=[ 2672], 50.00th=[ 2768], 60.00th=[ 2864],
     | 70.00th=[ 2992], 80.00th=[ 3120], 90.00th=[ 3376], 95.00th=[ 3632],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8256], 99.95th=[ 9280],
     | 99.99th=[12096]
    bw (KB  /s): min=    0, max=95616, per=20.01%, avg=90073.75, stdev=1808.52
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=2.17%, 4=95.58%, 10=2.20%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.42%, sys=5.94%, ctx=336153733, majf=0, minf=53946
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=202606853/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=148105: Thu Jan 14 23:20:48 2016
  read : io=3085.8GB, bw=449393KB/s, iops=28087, runt=7200003msec
    slat (usec): min=2, max=5768, avg=15.76, stdev=19.80
    clat (usec): min=54, max=125811, avg=2830.87, stdev=594.83
     lat (usec): min=310, max=125872, avg=2846.78, stdev=594.11
    clat percentiles (usec):
     |  1.00th=[ 1864],  5.00th=[ 2160], 10.00th=[ 2288], 20.00th=[ 2448],
     | 30.00th=[ 2576], 40.00th=[ 2672], 50.00th=[ 2768], 60.00th=[ 2864],
     | 70.00th=[ 2992], 80.00th=[ 3120], 90.00th=[ 3376], 95.00th=[ 3632],
     | 99.00th=[ 4640], 99.50th=[ 5600], 99.90th=[ 8256], 99.95th=[ 9152],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=96192, per=20.01%, avg=89905.18, stdev=1791.89
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=2.10%, 4=95.70%, 10=2.16%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.24%, sys=4.97%, ctx=340165167, majf=0, minf=60879
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.8%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=202226862/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=148110: Thu Jan 14 23:20:48 2016
  read : io=729406MB, bw=103738KB/s, iops=6483, runt=7200002msec
    slat (usec): min=2, max=3591, avg=20.50, stdev=24.33
    clat (usec): min=1, max=172472, avg=2444.84, stdev=580.40
     lat (usec): min=292, max=172488, avg=2465.56, stdev=579.35
    clat percentiles (usec):
     |  1.00th=[ 1496],  5.00th=[ 1784], 10.00th=[ 1912], 20.00th=[ 2064],
     | 30.00th=[ 2192], 40.00th=[ 2288], 50.00th=[ 2384], 60.00th=[ 2480],
     | 70.00th=[ 2608], 80.00th=[ 2736], 90.00th=[ 2992], 95.00th=[ 3216],
     | 99.00th=[ 4192], 99.50th=[ 5152], 99.90th=[ 7776], 99.95th=[ 8768],
     | 99.99th=[11456]
    bw (KB  /s): min=    0, max=28544, per=25.02%, avg=25951.37, stdev=645.70
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.07%
    lat (msec) : 2=14.64%, 4=84.06%, 10=1.18%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.58%, sys=1.85%, ctx=89052250, majf=0, minf=18285
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=46681996/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=148114: Thu Jan 14 23:20:48 2016
  read : io=752256MB, bw=106988KB/s, iops=6686, runt=7200001msec
    slat (usec): min=2, max=3419, avg=15.64, stdev=20.60
    clat (usec): min=15, max=111653, avg=2375.08, stdev=578.91
     lat (usec): min=271, max=111662, avg=2390.89, stdev=578.39
    clat percentiles (usec):
     |  1.00th=[ 1416],  5.00th=[ 1704], 10.00th=[ 1848], 20.00th=[ 1992],
     | 30.00th=[ 2128], 40.00th=[ 2224], 50.00th=[ 2320], 60.00th=[ 2416],
     | 70.00th=[ 2544], 80.00th=[ 2672], 90.00th=[ 2928], 95.00th=[ 3152],
     | 99.00th=[ 4080], 99.50th=[ 4960], 99.90th=[ 7712], 99.95th=[ 8640],
     | 99.99th=[11072]
    bw (KB  /s): min=    0, max=29796, per=25.02%, avg=26764.18, stdev=779.20
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
    lat (usec) : 750=0.03%, 1000=0.09%
    lat (msec) : 2=19.94%, 4=78.86%, 10=1.05%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=0.50%, sys=1.56%, ctx=91463274, majf=0, minf=15848
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48144370/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=3091.6GB, aggrb=450237KB/s, minb=450237KB/s, maxb=450237KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=3085.8GB, aggrb=449393KB/s, minb=449393KB/s, maxb=449393KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=729406MB, aggrb=103737KB/s, minb=103737KB/s, maxb=103737KB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=752256MB, aggrb=106987KB/s, minb=106987KB/s, maxb=106987KB/s, mint=7200001msec, maxt=7200001msec



# 
# Thu 14 Jan 2016 11:20:51 PM EST
# 8kB randread with SATA 6 jobs and 2 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 19473 IOPS | 155 MBps | 614 μsec Average Latency 
SATA-R5-1: 19043 IOPS | 152 MBps | 628 μsec Average Latency 
NVMe-R5-0: 14380 IOPS | 115 MBps | 554 μsec Average Latency 
NVMe-R5-1: 13970 IOPS | 111 MBps | 570 μsec Average Latency 

Totals
IOPS: 66866
MBPS: 533
Average Latency (μsec): 591

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=6): err= 0: pid=153286: Fri Jan 15 01:21:52 2016
  read : io=1069.8GB, bw=155787KB/s, iops=19473, runt=7200002msec
    slat (usec): min=2, max=15556, avg=11.03, stdev=10.91
    clat (usec): min=14, max=21607, avg=603.20, stdev=200.62
     lat (usec): min=122, max=21614, avg=614.37, stdev=200.16
    clat percentiles (usec):
     |  1.00th=[  398],  5.00th=[  458], 10.00th=[  474], 20.00th=[  494],
     | 30.00th=[  516], 40.00th=[  556], 50.00th=[  588], 60.00th=[  612],
     | 70.00th=[  628], 80.00th=[  652], 90.00th=[  732], 95.00th=[  812],
     | 99.00th=[ 1192], 99.50th=[ 1432], 99.90th=[ 3120], 99.95th=[ 3536],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=27168, per=16.67%, avg=25973.71, stdev=419.09
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=23.64%
    lat (usec) : 750=68.65%, 1000=5.36%
    lat (msec) : 2=2.02%, 4=0.29%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=0.86%, sys=2.35%, ctx=268269683, majf=0, minf=13457
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=140208026/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=6): err= 0: pid=153292: Fri Jan 15 01:21:52 2016
  read : io=1046.7GB, bw=152345KB/s, iops=19043, runt=7200001msec
    slat (usec): min=2, max=3475, avg=19.33, stdev=18.39
    clat (usec): min=12, max=38240, avg=608.48, stdev=195.50
     lat (usec): min=160, max=38271, avg=628.05, stdev=193.90
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  446], 10.00th=[  470], 20.00th=[  498],
     | 30.00th=[  532], 40.00th=[  564], 50.00th=[  588], 60.00th=[  612],
     | 70.00th=[  636], 80.00th=[  676], 90.00th=[  740], 95.00th=[  820],
     | 99.00th=[ 1192], 99.50th=[ 1368], 99.90th=[ 3056], 99.95th=[ 3536],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=27024, per=16.67%, avg=25400.12, stdev=419.85
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=20.39%
    lat (usec) : 750=70.90%, 1000=6.37%
    lat (msec) : 2=2.06%, 4=0.23%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=1.04%, sys=3.62%, ctx=260717794, majf=0, minf=17541
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=137110243/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=153298: Fri Jan 15 01:21:52 2016
  read : io=808925MB, bw=115047KB/s, iops=14380, runt=7200001msec
    slat (usec): min=2, max=3906, avg=10.73, stdev=10.38
    clat (usec): min=1, max=16925, avg=543.71, stdev=178.49
     lat (usec): min=103, max=16936, avg=554.58, stdev=177.90
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  374], 10.00th=[  422], 20.00th=[  466],
     | 30.00th=[  482], 40.00th=[  494], 50.00th=[  506], 60.00th=[  532],
     | 70.00th=[  580], 80.00th=[  612], 90.00th=[  660], 95.00th=[  748],
     | 99.00th=[ 1080], 99.50th=[ 1224], 99.90th=[ 2736], 99.95th=[ 3344],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=32768, per=25.01%, avg=28771.01, stdev=440.48
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.16%
    lat (usec) : 500=46.00%, 750=48.87%, 1000=3.50%
    lat (msec) : 2=1.29%, 4=0.15%, 10=0.03%, 20=0.01%
  cpu          : usr=0.89%, sys=2.46%, ctx=196814640, majf=0, minf=8720
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=103542453/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=153302: Fri Jan 15 01:21:52 2016
  read : io=785837MB, bw=111763KB/s, iops=13970, runt=7200001msec
    slat (usec): min=2, max=2156, avg=19.13, stdev=18.66
    clat (usec): min=0, max=23771, avg=551.19, stdev=180.84
     lat (usec): min=111, max=23782, avg=570.55, stdev=178.88
    clat percentiles (usec):
     |  1.00th=[  314],  5.00th=[  378], 10.00th=[  422], 20.00th=[  462],
     | 30.00th=[  482], 40.00th=[  498], 50.00th=[  524], 60.00th=[  556],
     | 70.00th=[  588], 80.00th=[  620], 90.00th=[  684], 95.00th=[  764],
     | 99.00th=[ 1080], 99.50th=[ 1224], 99.90th=[ 2768], 99.95th=[ 3376],
     | 99.99th=[ 6112]
    bw (KB  /s): min=    0, max=31216, per=25.01%, avg=27949.93, stdev=459.03
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.23%, 500=40.10%, 750=54.23%, 1000=3.93%
    lat (msec) : 2=1.34%, 4=0.14%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=1.14%, sys=3.78%, ctx=190370633, majf=0, minf=10708
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=100587127/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1069.8GB, aggrb=155786KB/s, minb=155786KB/s, maxb=155786KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=1046.7GB, aggrb=152344KB/s, minb=152344KB/s, maxb=152344KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=808925MB, aggrb=115047KB/s, minb=115047KB/s, maxb=115047KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=785837MB, aggrb=111763KB/s, minb=111763KB/s, maxb=111763KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 01:21:54 AM EST
# 8kB randread with SATA 4 jobs and 8 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 23919 IOPS | 191 MBps | 1336 μsec Average Latency 
SATA-R5-1: 23970 IOPS | 191 MBps | 1333 μsec Average Latency 
NVMe-R5-0: 12822 IOPS | 102 MBps | 1245 μsec Average Latency 
NVMe-R5-1: 12966 IOPS | 103 MBps | 1232 μsec Average Latency 

Totals
IOPS: 73677
MBPS: 587
Average Latency (μsec): 1286

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=159093: Fri Jan 15 03:22:55 2016
  read : io=1313.1GB, bw=191354KB/s, iops=23919, runt=7200001msec
    slat (usec): min=2, max=15641, avg=17.59, stdev=18.36
    clat (usec): min=17, max=95103, avg=1318.32, stdev=419.32
     lat (usec): min=268, max=95108, avg=1336.12, stdev=418.14
    clat percentiles (usec):
     |  1.00th=[  756],  5.00th=[  924], 10.00th=[  996], 20.00th=[ 1080],
     | 30.00th=[ 1144], 40.00th=[ 1208], 50.00th=[ 1256], 60.00th=[ 1304],
     | 70.00th=[ 1368], 80.00th=[ 1464], 90.00th=[ 1672], 95.00th=[ 1896],
     | 99.00th=[ 2992], 99.50th=[ 3760], 99.90th=[ 5472], 99.95th=[ 6816],
     | 99.99th=[ 8384]
    bw (KB  /s): min=    0, max=50496, per=25.01%, avg=47849.43, stdev=903.95
    lat (usec) : 20=0.01%, 100=0.01%, 250=0.01%, 500=0.03%, 750=0.89%
    lat (usec) : 1000=9.61%
    lat (msec) : 2=85.89%, 4=3.19%, 10=0.38%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.54%, sys=5.80%, ctx=294004093, majf=0, minf=15362
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172218586/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=159097: Fri Jan 15 03:22:55 2016
  read : io=1316.8GB, bw=191764KB/s, iops=23970, runt=7200002msec
    slat (usec): min=2, max=12044, avg=10.63, stdev=10.99
    clat (usec): min=53, max=78905, avg=1322.73, stdev=413.35
     lat (usec): min=223, max=78918, avg=1333.48, stdev=412.96
    clat percentiles (usec):
     |  1.00th=[  764],  5.00th=[  940], 10.00th=[ 1004], 20.00th=[ 1096],
     | 30.00th=[ 1144], 40.00th=[ 1208], 50.00th=[ 1256], 60.00th=[ 1320],
     | 70.00th=[ 1368], 80.00th=[ 1480], 90.00th=[ 1672], 95.00th=[ 1880],
     | 99.00th=[ 2896], 99.50th=[ 3728], 99.90th=[ 5472], 99.95th=[ 6816],
     | 99.99th=[ 8384]
    bw (KB  /s): min=    0, max=51520, per=25.01%, avg=47952.52, stdev=913.41
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.03%, 750=0.83%, 1000=8.95%
    lat (msec) : 2=86.73%, 4=3.08%, 10=0.37%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.26%, sys=3.83%, ctx=298242646, majf=0, minf=15759
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172588038/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=159101: Fri Jan 15 03:22:55 2016
  read : io=721269MB, bw=102581KB/s, iops=12822, runt=7200001msec
    slat (usec): min=2, max=2258, avg=18.48, stdev=19.00
    clat (usec): min=12, max=74060, avg=1227.02, stdev=383.93
     lat (usec): min=192, max=74078, avg=1245.73, stdev=382.80
    clat percentiles (usec):
     |  1.00th=[  684],  5.00th=[  844], 10.00th=[  924], 20.00th=[ 1004],
     | 30.00th=[ 1064], 40.00th=[ 1112], 50.00th=[ 1176], 60.00th=[ 1224],
     | 70.00th=[ 1288], 80.00th=[ 1384], 90.00th=[ 1560], 95.00th=[ 1768],
     | 99.00th=[ 2384], 99.50th=[ 3248], 99.90th=[ 5280], 99.95th=[ 6624],
     | 99.99th=[ 8096]
    bw (KB  /s): min=    0, max=27568, per=25.01%, avg=25655.41, stdev=537.04
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.08%
    lat (usec) : 750=1.99%, 1000=17.30%
    lat (msec) : 2=78.41%, 4=1.90%, 10=0.30%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.05%, sys=3.45%, ctx=171642530, majf=0, minf=10617
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=92322452/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=159105: Fri Jan 15 03:22:55 2016
  read : io=729360MB, bw=103731KB/s, iops=12966, runt=7200001msec
    slat (usec): min=2, max=2391, avg=10.86, stdev=11.83
    clat (usec): min=13, max=78199, avg=1221.29, stdev=379.68
     lat (usec): min=157, max=78209, avg=1232.27, stdev=379.30
    clat percentiles (usec):
     |  1.00th=[  684],  5.00th=[  844], 10.00th=[  924], 20.00th=[  996],
     | 30.00th=[ 1080], 40.00th=[ 1112], 50.00th=[ 1160], 60.00th=[ 1224],
     | 70.00th=[ 1272], 80.00th=[ 1368], 90.00th=[ 1544], 95.00th=[ 1752],
     | 99.00th=[ 2320], 99.50th=[ 3184], 99.90th=[ 5216], 99.95th=[ 6624],
     | 99.99th=[ 8160]
    bw (KB  /s): min=    0, max=28544, per=25.01%, avg=25943.57, stdev=511.39
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.10%
    lat (usec) : 750=2.05%, 1000=18.00%
    lat (msec) : 2=77.78%, 4=1.77%, 10=0.29%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.78%, sys=2.19%, ctx=175103881, majf=0, minf=10505
  IO depths    : 1=0.1%, 2=0.1%, 4=100.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=93358105/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=1313.1GB, aggrb=191353KB/s, minb=191353KB/s, maxb=191353KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1316.8GB, aggrb=191764KB/s, minb=191764KB/s, maxb=191764KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=721269MB, aggrb=102580KB/s, minb=102580KB/s, maxb=102580KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=729360MB, aggrb=103731KB/s, minb=103731KB/s, maxb=103731KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 03:22:58 AM EST
# 4kB randread with SATA 4 jobs and 2 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 2 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 14946 IOPS | 59 MBps | 533 μsec Average Latency 
SATA-R5-1: 15293 IOPS | 61 MBps | 521 μsec Average Latency 
NVMe-R5-0: 16683 IOPS | 66 MBps | 477 μsec Average Latency 
NVMe-R5-1: 17131 IOPS | 68 MBps | 465 μsec Average Latency 

Totals
IOPS: 64053
MBPS: 254
Average Latency (μsec): 499

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 16 processes

SATA-R5-01: (groupid=0, jobs=4): err= 0: pid=164280: Fri Jan 15 05:23:59 2016
  read : io=420366MB, bw=59785KB/s, iops=14946, runt=7200001msec
    slat (usec): min=2, max=2872, avg=17.37, stdev=15.99
    clat (usec): min=17, max=63016, avg=515.56, stdev=164.67
     lat (usec): min=129, max=63030, avg=533.17, stdev=163.09
    clat percentiles (usec):
     |  1.00th=[  322],  5.00th=[  370], 10.00th=[  402], 20.00th=[  446],
     | 30.00th=[  466], 40.00th=[  482], 50.00th=[  494], 60.00th=[  510],
     | 70.00th=[  532], 80.00th=[  572], 90.00th=[  620], 95.00th=[  668],
     | 99.00th=[ 1032], 99.50th=[ 1224], 99.90th=[ 2512], 99.95th=[ 2992],
     | 99.99th=[ 6432]
    bw (KB  /s): min=    0, max=15888, per=25.01%, avg=14950.88, stdev=278.18
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.14%, 500=54.04%
    lat (usec) : 750=43.24%, 1000=1.45%
    lat (msec) : 2=0.98%, 4=0.14%, 10=0.02%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.19%, sys=3.90%, ctx=201876807, majf=0, minf=7087
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=107613757/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=4): err= 0: pid=164284: Fri Jan 15 05:23:59 2016
  read : io=430142MB, bw=61176KB/s, iops=15293, runt=7200001msec
    slat (usec): min=1, max=1487, avg= 9.56, stdev= 8.25
    clat (usec): min=20, max=58889, avg=511.67, stdev=169.72
     lat (usec): min=111, max=58967, avg=521.36, stdev=169.30
    clat percentiles (usec):
     |  1.00th=[  334],  5.00th=[  370], 10.00th=[  406], 20.00th=[  454],
     | 30.00th=[  474], 40.00th=[  482], 50.00th=[  490], 60.00th=[  498],
     | 70.00th=[  516], 80.00th=[  548], 90.00th=[  612], 95.00th=[  652],
     | 99.00th=[ 1064], 99.50th=[ 1272], 99.90th=[ 2736], 99.95th=[ 3056],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=16544, per=25.01%, avg=15298.52, stdev=255.32
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.04%, 500=60.74%, 750=36.67%
    lat (usec) : 1000=1.35%
    lat (msec) : 2=0.98%, 4=0.20%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.94%, sys=2.45%, ctx=207199645, majf=0, minf=6561
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=110116459/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=164288: Fri Jan 15 05:23:59 2016
  read : io=469234MB, bw=66735KB/s, iops=16683, runt=7200001msec
    slat (usec): min=1, max=2864, avg=18.60, stdev=17.36
    clat (usec): min=1, max=68741, avg=458.66, stdev=145.63
     lat (usec): min=91, max=68748, avg=477.49, stdev=143.16
    clat percentiles (usec):
     |  1.00th=[  249],  5.00th=[  318], 10.00th=[  342], 20.00th=[  370],
     | 30.00th=[  394], 40.00th=[  430], 50.00th=[  458], 60.00th=[  478],
     | 70.00th=[  494], 80.00th=[  516], 90.00th=[  572], 95.00th=[  620],
     | 99.00th=[  900], 99.50th=[ 1064], 99.90th=[ 1464], 99.95th=[ 1960],
     | 99.99th=[ 5792]
    bw (KB  /s): min=    0, max=18168, per=25.01%, avg=16688.24, stdev=262.60
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=1.01%, 500=72.77%, 750=24.41%, 1000=1.14%
    lat (msec) : 2=0.62%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.30%, sys=4.40%, ctx=224403872, majf=0, minf=6949
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=120123844/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=164292: Fri Jan 15 05:23:59 2016
  read : io=481811MB, bw=68524KB/s, iops=17131, runt=7200001msec
    slat (usec): min=1, max=4036, avg= 9.92, stdev= 8.47
    clat (usec): min=10, max=61311, avg=455.32, stdev=138.76
     lat (usec): min=80, max=61340, avg=465.37, stdev=138.06
    clat percentiles (usec):
     |  1.00th=[  270],  5.00th=[  326], 10.00th=[  346], 20.00th=[  366],
     | 30.00th=[  386], 40.00th=[  426], 50.00th=[  462], 60.00th=[  478],
     | 70.00th=[  490], 80.00th=[  502], 90.00th=[  548], 95.00th=[  612],
     | 99.00th=[  884], 99.50th=[ 1048], 99.90th=[ 1400], 99.95th=[ 1768],
     | 99.99th=[ 5216]
    bw (KB  /s): min=    0, max=19120, per=25.01%, avg=17135.57, stdev=233.68
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.66%, 500=77.36%
    lat (usec) : 750=20.30%, 1000=1.06%
    lat (msec) : 2=0.57%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.00%, sys=2.73%, ctx=230905400, majf=0, minf=6942
  IO depths    : 1=0.1%, 2=100.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=123343649/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=420366MB, aggrb=59785KB/s, minb=59785KB/s, maxb=59785KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=430142MB, aggrb=61175KB/s, minb=61175KB/s, maxb=61175KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=469234MB, aggrb=66735KB/s, minb=66735KB/s, maxb=66735KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=481811MB, aggrb=68524KB/s, minb=68524KB/s, maxb=68524KB/s, mint=7200001msec, maxt=7200001msec



# 
# Fri 15 Jan 2016 05:24:01 AM EST
# 4kB randread with SATA 5 jobs and 16 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 


Summary
SATA-R5-0: 30735 IOPS | 122 MBps | 2601 μsec Average Latency 
SATA-R5-1: 30690 IOPS | 122 MBps | 2605 μsec Average Latency 
NVMe-R5-0: 6686 IOPS | 26 MBps | 2391 μsec Average Latency 
NVMe-R5-1: 6827 IOPS | 27 MBps | 2341 μsec Average Latency 

Totals
IOPS: 74938
MBPS: 297
Average Latency (μsec): 2484

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 18 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=169465: Fri Jan 15 07:25:03 2016
  read : io=864439MB, bw=122942KB/s, iops=30735, runt=7200003msec
    slat (usec): min=1, max=3339, avg=11.92, stdev=11.06
    clat (usec): min=143, max=1756.9K, avg=2589.61, stdev=1585.52
     lat (usec): min=212, max=1756.9K, avg=2601.66, stdev=1585.43
    clat percentiles (usec):
     |  1.00th=[ 1848],  5.00th=[ 2064], 10.00th=[ 2160], 20.00th=[ 2288],
     | 30.00th=[ 2384], 40.00th=[ 2480], 50.00th=[ 2544], 60.00th=[ 2608],
     | 70.00th=[ 2736], 80.00th=[ 2864], 90.00th=[ 3056], 95.00th=[ 3248],
     | 99.00th=[ 3760], 99.50th=[ 4080], 99.90th=[ 5600], 99.95th=[ 7136],
     | 99.99th=[10688]
    bw (KB  /s): min=    0, max=26056, per=20.01%, avg=24598.16, stdev=563.62
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=3.32%, 4=96.06%, 10=0.59%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.36%, sys=4.21%, ctx=377381825, majf=0, minf=21423
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=221296264/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=169470: Fri Jan 15 07:25:03 2016
  read : io=863158MB, bw=122760KB/s, iops=30690, runt=7200003msec
    slat (usec): min=2, max=3769, avg=16.34, stdev=15.82
    clat (usec): min=62, max=1758.1K, avg=2588.84, stdev=1533.56
     lat (usec): min=243, max=1758.1K, avg=2605.37, stdev=1533.34
    clat percentiles (usec):
     |  1.00th=[ 1816],  5.00th=[ 2064], 10.00th=[ 2160], 20.00th=[ 2288],
     | 30.00th=[ 2384], 40.00th=[ 2480], 50.00th=[ 2544], 60.00th=[ 2640],
     | 70.00th=[ 2736], 80.00th=[ 2864], 90.00th=[ 3024], 95.00th=[ 3216],
     | 99.00th=[ 3728], 99.50th=[ 4016], 99.90th=[ 5472], 99.95th=[ 7008],
     | 99.99th=[10560]
    bw (KB  /s): min=    0, max=26992, per=20.01%, avg=24561.71, stdev=548.56
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=3.50%, 4=95.95%, 10=0.52%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=1.60%, sys=5.53%, ctx=373106029, majf=0, minf=21822
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.7%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=220968390/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=4): err= 0: pid=169475: Fri Jan 15 07:25:03 2016
  read : io=188054MB, bw=26745KB/s, iops=6686, runt=7200001msec
    slat (usec): min=1, max=2248, avg=11.58, stdev=11.88
    clat (usec): min=39, max=1699.5K, avg=2379.76, stdev=1456.07
     lat (usec): min=189, max=1699.5K, avg=2391.48, stdev=1456.00
    clat percentiles (usec):
     |  1.00th=[ 1608],  5.00th=[ 1848], 10.00th=[ 1960], 20.00th=[ 2096],
     | 30.00th=[ 2160], 40.00th=[ 2256], 50.00th=[ 2352], 60.00th=[ 2416],
     | 70.00th=[ 2512], 80.00th=[ 2640], 90.00th=[ 2864], 95.00th=[ 3024],
     | 99.00th=[ 3536], 99.50th=[ 3792], 99.90th=[ 5216], 99.95th=[ 6816],
     | 99.99th=[10304]
    bw (KB  /s): min=    0, max=17688, per=25.02%, avg=6691.47, stdev=168.02
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.02%, 750=0.02%
    lat (usec) : 1000=0.03%
    lat (msec) : 2=13.52%, 4=86.09%, 10=0.32%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=0.48%, sys=1.24%, ctx=88780176, majf=0, minf=9539
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48141698/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=4): err= 0: pid=169479: Fri Jan 15 07:25:03 2016
  read : io=192035MB, bw=27312KB/s, iops=6827, runt=7200002msec
    slat (usec): min=1, max=6383, avg=18.03, stdev=17.40
    clat (usec): min=60, max=1668.7K, avg=2323.01, stdev=1415.23
     lat (usec): min=135, max=1668.7K, avg=2341.27, stdev=1415.06
    clat percentiles (usec):
     |  1.00th=[ 1512],  5.00th=[ 1768], 10.00th=[ 1880], 20.00th=[ 2008],
     | 30.00th=[ 2128], 40.00th=[ 2192], 50.00th=[ 2288], 60.00th=[ 2352],
     | 70.00th=[ 2480], 80.00th=[ 2608], 90.00th=[ 2800], 95.00th=[ 2992],
     | 99.00th=[ 3472], 99.50th=[ 3728], 99.90th=[ 5088], 99.95th=[ 6560],
     | 99.99th=[10048]
    bw (KB  /s): min=    0, max=11504, per=25.02%, avg=6833.18, stdev=182.36
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.04%
    lat (msec) : 2=18.62%, 4=81.02%, 10=0.28%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=0.65%, sys=1.78%, ctx=94474459, majf=0, minf=9137
  IO depths    : 1=0.1%, 2=0.1%, 4=100.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49161021/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=864439MB, aggrb=122942KB/s, minb=122942KB/s, maxb=122942KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=863158MB, aggrb=122760KB/s, minb=122760KB/s, maxb=122760KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=188054MB, aggrb=26745KB/s, minb=26745KB/s, maxb=26745KB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=192035MB, aggrb=27311KB/s, minb=27311KB/s, maxb=27311KB/s, mint=7200002msec, maxt=7200002msec