shenwei356/rush v0.0.6

rush v0.0.6

on GitHub

Changes:

• Avoid mixed line from multiple processes without loss of performance,
  e.g. the first half of a line is from one process
  and the last half of the line is from another process.
  similar with parallel --line-buffer

Benchmark:

===============================[ a few stdout ]===============================
--------------------------[ keep order: False ]--------------------------

commands:
seq 1 10 | gargs -p 4  "echo job:{}; seq 1 10" > t.gargs
seq 1 10 | rush -j 4  "echo job:{}; seq 1 10" > t.rush
seq 1 10 | parallel -j 4  "echo job:{}; seq 1 10" > t.parallel

time:
gargs: 0.0380618572235
parallel: 0.197314977646
rush: 0.0267391204834

md5sum:
2fd84d0e03f9d4402584914b2ba1dba9  t.gargs
6d82dfa19a9f0546bb376b2042139ded  t.parallel
b5aecd901fb76f894674ee4be7ae90b7  t.rush

---------------------------[ keep order: True ]---------------------------

commands:
seq 1 10 | gargs -p 4 -o "echo job:{}; seq 1 10" > t.gargs
seq 1 10 | rush -j 4 -k "echo job:{}; seq 1 10" > t.rush
seq 1 10 | parallel -j 4 -k "echo job:{}; seq 1 10" > t.parallel

time:
gargs: 0.0253150463104
parallel: 0.163496017456
rush: 0.023738861084

md5sum:
6d82dfa19a9f0546bb376b2042139ded  t.gargs
6d82dfa19a9f0546bb376b2042139ded  t.parallel
6d82dfa19a9f0546bb376b2042139ded  t.rush


==============================[ lots of stdout ]==============================
--------------------------[ keep order: False ]--------------------------

commands:
seq 1 10 | gargs -p 4  "echo job:{}; seq 1 1000000" > t.gargs
seq 1 10 | rush -j 4  "echo job:{}; seq 1 1000000" > t.rush
seq 1 10 | parallel -j 4  "echo job:{}; seq 1 1000000" > t.parallel

time:
gargs: 0.22896194458
parallel: 0.301723957062
rush: 0.243381023407

md5sum:
edc13e1bcc9700fccfa524ef4c078688  t.gargs
509086b10292759096d6db278bb2add8  t.parallel
8b0d20a76dfa9db701ae850968796478  t.rush

---------------------------[ keep order: True ]---------------------------

commands:
seq 1 10 | gargs -p 4 -o "echo job:{}; seq 1 1000000" > t.gargs
seq 1 10 | rush -j 4 -k "echo job:{}; seq 1 1000000" > t.rush
seq 1 10 | parallel -j 4 -k "echo job:{}; seq 1 1000000" > t.parallel

time:
gargs: 0.231871128082
parallel: 0.297834157944
rush: 0.237619161606

md5sum:
509086b10292759096d6db278bb2add8  t.gargs
509086b10292759096d6db278bb2add8  t.parallel
509086b10292759096d6db278bb2add8  t.rush


==========================[ a few stdout & run long ]==========================
--------------------------[ keep order: False ]--------------------------

commands:
seq 1 10 | gargs -p 4  "echo job:{}; sleep 2; seq 1 10" > t.gargs
seq 1 10 | rush -j 4  "echo job:{}; sleep 2; seq 1 10" > t.rush
seq 1 10 | parallel -j 4  "echo job:{}; sleep 2; seq 1 10" > t.parallel

time:
gargs: 6.02603578568
parallel: 6.1913690567
rush: 6.02895188332

md5sum:
053741317f4adcee133daf736b58cd8e  t.gargs
6d82dfa19a9f0546bb376b2042139ded  t.parallel
b336503c7faefd2602a356d99f3647c8  t.rush

---------------------------[ keep order: True ]---------------------------

commands:
seq 1 10 | gargs -p 4 -o "echo job:{}; sleep 2; seq 1 10" > t.gargs
seq 1 10 | rush -j 4 -k "echo job:{}; sleep 2; seq 1 10" > t.rush
seq 1 10 | parallel -j 4 -k "echo job:{}; sleep 2; seq 1 10" > t.parallel

time:
gargs: 6.03132200241
parallel: 6.19274401665
rush: 6.03151106834

md5sum:
6d82dfa19a9f0546bb376b2042139ded  t.gargs
6d82dfa19a9f0546bb376b2042139ded  t.parallel
6d82dfa19a9f0546bb376b2042139ded  t.rush


=========================[ lots of stdout & run long ]=========================
--------------------------[ keep order: False ]--------------------------

commands:
seq 1 10 | gargs -p 4  "echo job:{}; sleep 2; seq 1 1000000" > t.gargs
seq 1 10 | rush -j 4  "echo job:{}; sleep 2; seq 1 1000000" > t.rush
seq 1 10 | parallel -j 4  "echo job:{}; sleep 2; seq 1 1000000" > t.parallel

time:
gargs: 6.15697097778
parallel: 6.33681201935
rush: 6.17427492142

md5sum:
e1987d0d7528b9607ac848f7cf81c630  t.gargs
509086b10292759096d6db278bb2add8  t.parallel
bce691db6deafe5bfc37bd99fe8aea6b  t.rush

---------------------------[ keep order: True ]---------------------------

commands:
seq 1 10 | gargs -p 4 -o "echo job:{}; sleep 2; seq 1 1000000" > t.gargs
seq 1 10 | rush -j 4 -k "echo job:{}; sleep 2; seq 1 1000000" > t.rush
seq 1 10 | parallel -j 4 -k "echo job:{}; sleep 2; seq 1 1000000" > t.parallel

time:
gargs: 6.16847395897
parallel: 6.28346395493
rush: 6.16706514359

md5sum:
509086b10292759096d6db278bb2add8  t.gargs
509086b10292759096d6db278bb2add8  t.parallel
509086b10292759096d6db278bb2add8  t.rush