diff --git a/run-regression.rb b/run-regression.rb new file mode 100755 index 0000000..b241a01 --- /dev/null +++ b/run-regression.rb @@ -0,0 +1,345 @@ +#!/usr/local/bin/ruby +#---------------------------------------------------------------------- +# run-regression.rb +#---------------------------------------------------------------------- +# Copyright (c) 2007,2008,2009 by Johan Holmberg. All rights reserved. +#---------------------------------------------------------------------- +# @(#) $Id$ +#---------------------------------------------------------------------- + +require "rbconfig" +require "dbm" +require "digest/md5" + +#---------------------------------------------------------------------- + +TMPFILES = [ + "tmp-cmdtest-2", + "tmp-cmdtest-2/tmp-command.sh", + "tmp-cmdtest-2/tmp-stderr.log", + "tmp-cmdtest-2/tmp-stdout.log", + "tmp-cmdtest-2/workdir", + "tmp-cmdtest-2/workdir/TIMESTAMP" +] + +class ActualTcRegression + + def initialize + @count = 0 + @f = File.open("TC_actual-regression.rb", "w") + + @f.puts [ + "class TC_cmdtest < Cmdtest::Testcase", + "", + " def setup", + TMPFILES.map {|file| " ignore_file #{file.inspect}" }, + " end", + "", + ] + + end + + def _list_elements(level, lines) + lines.map do |line| + " " * level + line.inspect + "," + end + end + + def _indent(level, lines) + lines.map do |line| + " " * level + line + end + end + + def _gen_count + @count += 1 + end + + def _comment_str(lines) + lines.each do |line| + if line =~ /^#/ # .. + res = line.sub(/^#\s*/, "") + return res + end + end + return "no comment given" + end + + def add(prefix, code, stdout) + @f.puts [ + " def test_#{_gen_count}", + " create_file \"TC_tmp.rb\", [", + " 'class TC_cmdtest < Cmdtest::Testcase',", + " ' def test_foo',", + _list_elements(3, _indent(2, prefix)), + _list_elements(3, _indent(2, code)), + " ' end',", + " 'end',", + " ]", + "", + " cmd 'cmdtest.rb --quiet' do", + " comment #{_comment_str(code).inspect}", + " stdout_equal [", + _list_elements(4, stdout), + " ]", + " end", + " end", + "", + ].flatten + end + + def write + @f.puts [ + "", + "end", + "", + ] + + @f.close + end +end + +#---------------------------------------------------------------------- + +class ActualRegression + + def initialize + @o2 = ActualTcRegression.new + + @f = File.open("actual-regression.rb", "w") + end + + def add(prefix, code, stdout) + prefix = prefix.map {|line| line.chomp } + code = code .map {|line| line.chomp } + stdout = stdout.map {|line| line.chomp } + + @o2.add(prefix,code,stdout) + + @f.puts prefix + @f.puts "#" + "-" * 35 + @f.puts code + @f.puts "# stdout begin" + @f.puts stdout.map {|line| "# %s" % [line] } + @f.puts "# stdout end" + end + + def write + @o2.write + + @f.close + end +end + +#---------------------------------------------------------------------- + +class RegressionData + + def initialize(files) + @lines = files.map {|file| File.readlines(file) }.flatten + @i = 0 + end + + def eof? + j = @i + while j < @lines.size && @lines[j].strip.empty? + j += 1 + end + return j >= @lines.size + end + + def skip_to(pattern) + # ignore return value + get_to(pattern) + end + + def _show_line(i) + puts "%d: %s" % [ + i, + @lines[i], + ] + end + + def get_to(pattern) + res = [] + i1 = @i + while ! eof? && @lines[@i] !~ pattern + _show_line(@i) if $opt_verbose + res << @lines[@i] + @i += 1 + end + if eof? + puts "Error: looking at:" + _show_line(i1) + raise "eof looking for #{pattern}" + end + # get past matching line + @i += 1 + return res + end +end + +#---------------------------------------------------------------------- + +SystemResult = Struct.new(:status, :stdout, :stderr) + +def my_system(cmd) + full_cmd = "#{cmd} > tmp-stdout 2> tmp-stderr" + #puts "+ #{full_cmd}" + ok = system full_cmd + status = $?.exitstatus + stdout = File.readlines("tmp-stdout") + stderr = File.readlines("tmp-stderr") + if status != 0 + puts "INTERNAL ERROR:" + puts "STDOUT:" + puts stdout + puts "STDERR:" + puts stderr + exit 1 + end + #p [:my_system, cmd, status, stdout, stderr] + return SystemResult.new(status, stdout, stderr) +end + +#---------------------------------------------------------------------- + +def indented(lines) + lines.map {|line| " " + line.to_s } +end + +#---------------------------------------------------------------------- + +def matching_arrays(a,b) + return false if a.size != b.size + a.each_index do |i| + return false unless a[i] === b[i] + end + return true +end + +#---------------------------------------------------------------------- + +$opt_verbose = false +$opt_only = nil +$opt_remember = false + +while /^-/ =~ ARGV[0] + arg = ARGV.shift + case arg + when "-v" + $opt_verbose = true + when "-r" + $opt_remember = true + when /^--only=(\d+)$/ + $opt_only = $1.to_i + else + puts "Error: unknown option: #{arg}" + exit 1 + end +end + +ENV["PATH"] = (File.expand_path("t/bin") + + Config::CONFIG["PATH_SEPARATOR"] + + ENV["PATH"]) + +files = ARGV.empty? ? Dir.glob("t/*.rb") : ARGV +rd = RegressionData.new(files) + +tests = [] + +while ! rd.eof? + prefix = rd.get_to( /^#----------/ ) + code = rd.get_to( /^# stdout begin/ ) + stdout = rd.get_to( /^# stdout end/ ).map do |line| + if line =~ /^# (.*\n)/ #.. + $1 + elsif line =~ /^#\/(.*)/ #.. + Regexp.new($1) + else + line # should not occur + end + end + tests << [prefix, code, stdout] +end + +puts "###" +puts "### found %d tests in file." % [tests.size] +puts "###" + +act = ActualRegression.new + +already_ok = DBM.open("already_ok") +if ! $opt_remember + already_ok.clear +end + +errors = 0 +iii = -1 +for prefix, code, stdout in tests + iii += 1 + next if $opt_only && iii != $opt_only + + digest = Digest::MD5.hexdigest([prefix,code,stdout].flatten.join) + if already_ok.has_key?(digest) + #puts "### #{iii}: %s ... [[cahced]]" % [code[0].chomp] + next + end + + if code.join("\n") =~ /SKIP \s+ (\S+)/x + skip = $1 + if RUBY_PLATFORM.index(skip) + puts "### #{iii}: SKIP: %s ..." % [code[0].chomp] + act.add(prefix, code, stdout) + next + end + end + + puts "### #{iii}: %s ..." % [code[0].chomp] + + lines = [] + #lines << "require 'Test/Cmd'" + lines << "" + lines << "class TC_foo < Cmdtest::Testcase" + lines << "" + lines << " def test_foo" + lines << code + lines << " end" + lines << "" + lines << "end" + lines << "" + lines.flatten! + + File.open("CMDTEST_tmp_regression.rb", "w") do |f| + f.puts lines + end + + res = my_system "ruby -w bin/cmdtest.rb --quiet --ruby_s CMDTEST_tmp_regression.rb" + if res.status != 0 + puts "ERROR: non-zero exit from:" + puts lines + exit 1 + end + if matching_arrays(stdout, res.stdout) + already_ok[digest] = true + else + puts "ERROR: unexpected STDOUT:" + puts "ACTUAL:" + puts indented(res.stdout) + puts "EXPECTED:" + puts indented(stdout) + errors += 1 + end + + act.add(prefix, code, res.stdout) +end + +act.write + +puts +if errors == 0 + puts "### all tests OK" +else + puts "--- error in #{errors} tests" +end +puts diff --git a/t/00-assert.rb b/t/00-assert.rb new file mode 100644 index 0000000..90e45f3 --- /dev/null +++ b/t/00-assert.rb @@ -0,0 +1,38 @@ +#====================================================================== +# test assert + +#----------------------------------- +# assert -- correct + +cmd "true.rb" do + assert true +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# assert -- incorrect + +cmd "true.rb" do + assert false +end + +# stdout begin +# ### true.rb +# --- ERROR: assertion failed +# stdout end + +#----------------------------------- +# assert -- incorrect with msg + +cmd "true.rb" do + assert false, "got false" +end + +# stdout begin +# ### true.rb +# --- ERROR: assertion: got false +# stdout end + diff --git a/t/00-exit_nonzero.rb b/t/00-exit_nonzero.rb new file mode 100644 index 0000000..d9947f1 --- /dev/null +++ b/t/00-exit_nonzero.rb @@ -0,0 +1,37 @@ +#====================================================================== +# test exit_nonzero + +#----------------------------------- +# exit_nonzero -- correct + +cmd "false.rb" do + exit_nonzero +end + +# stdout begin +# ### false.rb +# stdout end + +#----------------------------------- +# exit_nonzero -- correct 18 + +cmd "exit.rb 18" do + exit_nonzero +end + +# stdout begin +# ### exit.rb 18 +# stdout end + +#----------------------------------- +# exit_nonzero -- incorrect + +cmd "true.rb" do + exit_nonzero +end + +# stdout begin +# ### true.rb +# --- ERROR: expected nonzero exit status +# stdout end + diff --git a/t/00-exit_status.rb b/t/00-exit_status.rb new file mode 100644 index 0000000..3d63947 --- /dev/null +++ b/t/00-exit_status.rb @@ -0,0 +1,49 @@ +#====================================================================== +# test exit_status + +#----------------------------------- +# exit_status -- correct 0 + +cmd "true.rb" do + exit_status 0 +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# exit_status -- incorrect 0 + +cmd "false.rb" do + exit_status 0 +end + +# stdout begin +# ### false.rb +# --- ERROR: expected 0 exit status, got 1 +# stdout end + +#----------------------------------- +# exit_status -- correct 18 + +cmd "exit.rb 18" do + exit_status 18 +end + +# stdout begin +# ### exit.rb 18 +# stdout end + +#----------------------------------- +# exit_status -- incorrect 18 + +cmd "exit.rb 10" do + exit_status 18 +end + +# stdout begin +# ### exit.rb 10 +# --- ERROR: expected 18 exit status, got 10 +# stdout end + diff --git a/t/00-exit_zero.rb b/t/00-exit_zero.rb new file mode 100644 index 0000000..1c6dd37 --- /dev/null +++ b/t/00-exit_zero.rb @@ -0,0 +1,38 @@ +#====================================================================== +# test exit_zero + +#----------------------------------- +# exit_zero -- correct + +cmd "true.rb" do + exit_zero +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# exit_zero -- incorrect + +cmd "false.rb" do + exit_zero +end + +# stdout begin +# ### false.rb +# --- ERROR: expected zero exit status, got 1 +# stdout end + +#----------------------------------- +# exit_zero -- incorrect 18 + +cmd "exit.rb 18" do + exit_zero +end + +# stdout begin +# ### exit.rb 18 +# --- ERROR: expected zero exit status, got 18 +# stdout end + diff --git a/t/00-file_equal.rb b/t/00-file_equal.rb new file mode 100644 index 0000000..f40adc2 --- /dev/null +++ b/t/00-file_equal.rb @@ -0,0 +1,161 @@ +#====================================================================== +# test file_equal + +#----------------------------------- +# file_equal -- correct "" + +File.open("foo", "w") {} + +cmd "true.rb" do + file_equal "foo", "" +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file_equal -- incorrect "" + +File.open("foo", "w") {|f| f.puts "hello world" } + +cmd "true.rb" do + file_equal "foo", "" +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong file 'foo' +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# file_equal -- correct [] + +File.open("foo", "w") {} + +cmd "true.rb" do + file_equal "foo", [] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file_equal -- incorrect [] + +File.open("foo", "w") {|f| f.puts "hello world" } + +cmd "true.rb" do + file_equal "foo", [] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong file 'foo' +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# file_equal -- correct [ "hello world" ] + +File.open("foo", "w") {|f| f.puts "hello world" } + +cmd "true.rb" do + file_equal "foo", [ "hello world" ] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file_equal -- incorrect [ "hello world" ] + +File.open("foo", "w") {} + +cmd "true.rb" do + file_equal "foo", [ "hello world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong file 'foo' +# --- actual: [[empty]] +# --- expect: hello world +# stdout end + +#----------------------------------- +# file_equal -- correct [ "hello", "world" ] + +File.open("foo", "w") {|f| f.puts "hello"; f.puts "world" } + +cmd "true.rb" do + file_equal "foo", [ "hello", "world" ] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file_equal -- incorrect [ "hello", "world" ] + +File.open("foo", "w") {} + +cmd "true.rb" do + file_equal "foo", [ "hello", "world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong file 'foo' +# --- actual: [[empty]] +# --- expect: hello +# --- world +# stdout end + +#----------------------------------- +# file_equal -- non-existing file + +cmd "true.rb" do + file_equal "foo", "" +end + +# stdout begin +# ### true.rb +# --- ERROR: no such file: 'foo' +# stdout end + +#----------------------------------- +# file_equal -- file is directory + +Dir.mkdir "foo" + +cmd "true.rb" do + file_equal "foo", "" +end + +# stdout begin +# ### true.rb +# --- ERROR: is a directory: 'foo' +# stdout end + +#----------------------------------- +# file_equal -- other error +# SKIP mswin32 + +File.symlink "foo", "foo" + +cmd "true.rb" do + file_equal "foo", "" +end + +# stdout begin +# ### true.rb +# --- ERROR: error reading file: 'foo' +# stdout end + diff --git a/t/00-file_not_equal.rb b/t/00-file_not_equal.rb new file mode 100644 index 0000000..4709d00 --- /dev/null +++ b/t/00-file_not_equal.rb @@ -0,0 +1,32 @@ +#====================================================================== +# test file_not_equal + +#----------------------------------- +# file_not_equal -- correct "" + +File.open("foo", "w") {|f| f.puts "hello" } + +cmd "true.rb" do + file_not_equal "foo", "" +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file_not_equal -- incorrect "" + +File.open("foo", "w") {} + +cmd "true.rb" do + file_not_equal "foo", "" +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong file 'foo' +# --- actual: [[empty]] +# --- expect: [[empty]] +# stdout end + diff --git a/t/00-ignore_file.rb b/t/00-ignore_file.rb new file mode 100644 index 0000000..5464999 --- /dev/null +++ b/t/00-ignore_file.rb @@ -0,0 +1,59 @@ +#====================================================================== +# test ignore_file + +#----------------------------------- +# ignore_file -- actually created + +ignore_file "bbb" + +cmd "touch.rb bbb" do +end + +# stdout begin +# ### touch.rb bbb +# stdout end + +#----------------------------------- +# ignore_file -- not created + +ignore_file "aaa" + +cmd "touch.rb bbb" do +end + +# stdout begin +# ### touch.rb bbb +# --- ERROR: created files +# --- actual: ["bbb"] +# --- expect: [] +# stdout end + +#----------------------------------- +# ignore_file -- in subdir + ok + +ignore_file "dir/bbb" +Dir.mkdir "dir" + +cmd "touch.rb dir/bbb" do +end + +# stdout begin +# ### touch.rb dir/bbb +# stdout end + +#----------------------------------- +# ignore_file -- in subdir + error + +ignore_file "bbb" +Dir.mkdir "dir" + +cmd "touch.rb dir/bbb" do +end + +# stdout begin +# ### touch.rb dir/bbb +# --- ERROR: created files +# --- actual: ["dir/bbb"] +# --- expect: [] +# stdout end + diff --git a/t/00-import_file.rb b/t/00-import_file.rb new file mode 100644 index 0000000..3bb702e --- /dev/null +++ b/t/00-import_file.rb @@ -0,0 +1,33 @@ +#====================================================================== +# test import_file + +#----------------------------------- +# import_file -- different dirs + +import_file "file1.txt", "qwerty1.txt" +import_file "file2.txt", "subdir/qwerty2.txt" + +cmd "cat.rb qwerty1.txt subdir/qwerty2.txt > qwerty3.txt" do + created_files "qwerty3.txt" +end + +# stdout begin +# ### cat.rb qwerty1.txt subdir/qwerty2.txt > qwerty3.txt +# stdout end + +#----------------------------------- +# import_file -- after chdir + +Dir.mkdir("dir") +Dir.chdir("dir") +import_file "file1.txt", "qwerty1.txt" +import_file "file2.txt", "subdir/qwerty2.txt" + +cmd "cat.rb qwerty1.txt subdir/qwerty2.txt > qwerty3.txt" do + created_files "dir/qwerty3.txt" +end + +# stdout begin +# ### cat.rb qwerty1.txt subdir/qwerty2.txt > qwerty3.txt +# stdout end + diff --git a/t/00-stderr_equal.rb b/t/00-stderr_equal.rb new file mode 100644 index 0000000..cd692bf --- /dev/null +++ b/t/00-stderr_equal.rb @@ -0,0 +1,104 @@ +#====================================================================== +# test stderr_equal + +#----------------------------------- +# stderr_equal -- correct "" + +cmd "true.rb" do + stderr_equal "" +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# stderr_equal -- incorrect "" + +cmd "echo.rb hello world >&2" do + stderr_equal "" +end + +# stdout begin +# ### echo.rb hello world >&2 +# --- ERROR: wrong stderr +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stderr_equal -- correct [] + +cmd "true.rb" do + stderr_equal [] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# stderr_equal -- incorrect [] + +cmd "echo.rb hello world >&2" do + stderr_equal [] +end + +# stdout begin +# ### echo.rb hello world >&2 +# --- ERROR: wrong stderr +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stderr_equal -- correct [ "hello world" ] + +cmd "echo.rb hello world >&2" do + stderr_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb hello world >&2 +# stdout end + +#----------------------------------- +# stderr_equal -- incorrect [ "hello world" ] + +cmd "true.rb" do + stderr_equal [ "hello world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stderr +# --- actual: [[empty]] +# --- expect: hello world +# stdout end + +#----------------------------------- +# stderr_equal -- correct [ "hello", "world" ] + +cmd "echo.rb hello >&2 && echo.rb world >&2" do + stderr_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello >&2 && echo.rb world >&2 +# stdout end + +#----------------------------------- +# stderr_equal -- incorrect [ "hello", "world" ] + +cmd "true.rb" do + stderr_equal [ "hello", "world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stderr +# --- actual: [[empty]] +# --- expect: hello +# --- world +# stdout end + diff --git a/t/00-stderr_not_equal.rb b/t/00-stderr_not_equal.rb new file mode 100644 index 0000000..fc4fe6f --- /dev/null +++ b/t/00-stderr_not_equal.rb @@ -0,0 +1,105 @@ +#====================================================================== +# test stderr_not_equal + +#----------------------------------- +# stderr_not_equal -- correct "" + +cmd "echo.rb hello >&2" do + stderr_not_equal "" +end + +# stdout begin +# ### echo.rb hello >&2 +# stdout end + +#----------------------------------- +# stderr_not_equal -- incorrect "" + +cmd "true.rb" do + stderr_not_equal "" +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stderr +# --- actual: [[empty]] +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stderr_not_equal -- correct [] + +cmd "echo.rb hello >&2" do + stderr_not_equal [] +end + +# stdout begin +# ### echo.rb hello >&2 +# stdout end + +#----------------------------------- +# stderr_not_equal -- incorrect [] + +cmd "true.rb" do + stderr_not_equal [] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stderr +# --- actual: [[empty]] +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stderr_not_equal -- correct [ "hello world" ] + +cmd "echo.rb not hello world >&2" do + stderr_not_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb not hello world >&2 +# stdout end + +#----------------------------------- +# stderr_not_equal -- incorrect [ "hello world" ] + +cmd "echo.rb hello world >&2" do + stderr_not_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb hello world >&2 +# --- ERROR: wrong stderr +# --- actual: hello world +# --- expect: hello world +# stdout end + +#----------------------------------- +# stderr_not_equal -- correct [ "hello", "world" ] + +cmd "echo.rb hello world >&2" do + stderr_not_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello world >&2 +# stdout end + +#----------------------------------- +# stderr_not_equal -- incorrect [ "hello", "world" ] + +cmd "echo.rb hello >&2 && echo.rb world >&2" do + stderr_not_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello >&2 && echo.rb world >&2 +# --- ERROR: wrong stderr +# --- actual: hello +# --- world +# --- expect: hello +# --- world +# stdout end + diff --git a/t/00-stdout_equal.rb b/t/00-stdout_equal.rb new file mode 100644 index 0000000..608efda --- /dev/null +++ b/t/00-stdout_equal.rb @@ -0,0 +1,104 @@ +#====================================================================== +# test stdout_equal + +#----------------------------------- +# stdout_equal -- correct "" + +cmd "true.rb" do + stdout_equal "" +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# stdout_equal -- incorrect "" + +cmd "echo.rb hello world" do + stdout_equal "" +end + +# stdout begin +# ### echo.rb hello world +# --- ERROR: wrong stdout +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stdout_equal -- correct [] + +cmd "true.rb" do + stdout_equal [] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# stdout_equal -- incorrect [] + +cmd "echo.rb hello world" do + stdout_equal [] +end + +# stdout begin +# ### echo.rb hello world +# --- ERROR: wrong stdout +# --- actual: hello world +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stdout_equal -- correct [ "hello world" ] + +cmd "echo.rb hello world" do + stdout_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb hello world +# stdout end + +#----------------------------------- +# stdout_equal -- incorrect [ "hello world" ] + +cmd "true.rb" do + stdout_equal [ "hello world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stdout +# --- actual: [[empty]] +# --- expect: hello world +# stdout end + +#----------------------------------- +# stdout_equal -- correct [ "hello", "world" ] + +cmd "echo.rb hello && echo.rb world" do + stdout_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello && echo.rb world +# stdout end + +#----------------------------------- +# stdout_equal -- incorrect [ "hello", "world" ] + +cmd "true.rb" do + stdout_equal [ "hello", "world" ] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stdout +# --- actual: [[empty]] +# --- expect: hello +# --- world +# stdout end + diff --git a/t/00-stdout_not_equal.rb b/t/00-stdout_not_equal.rb new file mode 100644 index 0000000..a8ed4b4 --- /dev/null +++ b/t/00-stdout_not_equal.rb @@ -0,0 +1,105 @@ +#====================================================================== +# test stdout_not_equal + +#----------------------------------- +# stdout_not_equal -- correct "" + +cmd "echo.rb hello" do + stdout_not_equal "" +end + +# stdout begin +# ### echo.rb hello +# stdout end + +#----------------------------------- +# stdout_not_equal -- incorrect "" + +cmd "true.rb" do + stdout_not_equal "" +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stdout +# --- actual: [[empty]] +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stdout_not_equal -- correct [] + +cmd "echo.rb hello" do + stdout_not_equal [] +end + +# stdout begin +# ### echo.rb hello +# stdout end + +#----------------------------------- +# stdout_not_equal -- incorrect [] + +cmd "true.rb" do + stdout_not_equal [] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stdout +# --- actual: [[empty]] +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# stdout_not_equal -- correct [ "hello world" ] + +cmd "echo.rb not hello world" do + stdout_not_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb not hello world +# stdout end + +#----------------------------------- +# stdout_not_equal -- incorrect [ "hello world" ] + +cmd "echo.rb hello world" do + stdout_not_equal [ "hello world" ] +end + +# stdout begin +# ### echo.rb hello world +# --- ERROR: wrong stdout +# --- actual: hello world +# --- expect: hello world +# stdout end + +#----------------------------------- +# stdout_not_equal -- correct [ "hello", "world" ] + +cmd "echo.rb hello world" do + stdout_not_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello world +# stdout end + +#----------------------------------- +# stdout_not_equal -- incorrect [ "hello", "world" ] + +cmd "echo.rb hello && echo.rb world" do + stdout_not_equal [ "hello", "world" ] +end + +# stdout begin +# ### echo.rb hello && echo.rb world +# --- ERROR: wrong stdout +# --- actual: hello +# --- world +# --- expect: hello +# --- world +# stdout end + diff --git a/t/01-misc.rb b/t/01-misc.rb new file mode 100644 index 0000000..b8a7fe1 --- /dev/null +++ b/t/01-misc.rb @@ -0,0 +1,643 @@ +#====================================================================== + +#----------------------------------- +# Ruby script called with "ruby -S" + +cmd "echo.rb this is a line" do + stdout_equal "this is a line\n" +end + +# stdout begin +# ### echo.rb this is a line +# stdout end + +#----------------------------------- +# actual "false.rb" will give error + +cmd "true.rb" do +end + +cmd "false.rb" do +end + +# stdout begin +# ### true.rb +# ### false.rb +# --- ERROR: expected zero exit status, got 1 +# stdout end + +#----------------------------------- +# actual "false.rb" will give error + +cmd "false.rb" do +end + +# stdout begin +# ### false.rb +# --- ERROR: expected zero exit status, got 1 +# stdout end + +#----------------------------------- +# another non-zero exit will give error + +cmd "exit.rb 18" do +end + +# stdout begin +# ### exit.rb 18 +# --- ERROR: expected zero exit status, got 18 +# stdout end + +#----------------------------------- +# actual STDOUT will give error + +cmd "echo.rb a line on stdout" do +end + +# stdout begin +# ### echo.rb a line on stdout +# --- ERROR: wrong stdout +# --- actual: a line on stdout +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# actual STDERR will give error + +cmd "echo.rb a line on stderr 1>&2" do +end + +# stdout begin +# ### echo.rb a line on stderr 1>&2 +# --- ERROR: wrong stderr +# --- actual: a line on stderr +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# existing files is OK + +File.open("before1", "w") {} +File.open("before2", "w") {} +cmd "true.rb" do +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# actual created file will give error + +cmd "echo.rb content > a-new-file" do +end + +# stdout begin +# ### echo.rb content > a-new-file +# --- ERROR: created files +# --- actual: ["a-new-file"] +# --- expect: [] +# stdout end + +#----------------------------------- +# several actual created files will give error + +File.open("before1", "w") {} +File.open("before2", "w") {} +cmd "echo.rb x > a && echo.rb x > b" do +end + +# stdout begin +# ### echo.rb x > a && echo.rb x > b +# --- ERROR: created files +# --- actual: ["a", "b"] +# --- expect: [] +# stdout end + +#----------------------------------- +# actual removed file will give error + +File.open("before", "w") {} +cmd "rm.rb before" do +end + +# stdout begin +# ### rm.rb before +# --- ERROR: removed files +# --- actual: ["before"] +# --- expect: [] +# stdout end + +#----------------------------------- +# several actual removed files will give error + +File.open("before1", "w") {} +File.open("before2", "w") {} +File.open("before3", "w") {} +cmd "rm.rb before1 before2" do +end + +# stdout begin +# ### rm.rb before1 before2 +# --- ERROR: removed files +# --- actual: ["before1", "before2"] +# --- expect: [] +# stdout end + +#----------------------------------- +# actual changed files will give error + +# NOTE: order of writing/testing is important below + +File.open("changed1", "w") {} +File.open("changed2", "w") {} + +File.open("script.rb", "w") do |f| + f.puts "t1 = File.mtime('changed2')" + f.puts "while File.mtime('changed2') == t1" + f.puts " File.open('changed2', 'w') {|f| f.puts 111 }" + f.puts " File.open('changed1', 'w') {|f| f.puts 111 }" + f.puts "end" +end + +cmd "ruby script.rb" do +end + +# stdout begin +# ### ruby script.rb +# --- ERROR: changed files +# --- actual: ["changed1", "changed2"] +# --- expect: [] +# stdout end + +#----------------------------------- +# mix of actual created/removed files will give error + +File.open("before1", "w") {} +File.open("before2", "w") {} +File.open("before3", "w") {} +cmd "rm.rb before1 before2 && echo.rb x > a && echo.rb x > b" do +end + +# stdout begin +# ### rm.rb before1 before2 && echo.rb x > a && echo.rb x > b +# --- ERROR: created files +# --- actual: ["a", "b"] +# --- expect: [] +# --- ERROR: removed files +# --- actual: ["before1", "before2"] +# --- expect: [] +# stdout end + +#----------------------------------- +# mix of "all" errros + +File.open("before1", "w") {} +File.open("before2", "w") {} +File.open("before3", "w") {} +File.open("script.rb", "w") do |f| + f.puts "File.unlink 'before1'" + f.puts "File.unlink 'before2'" + f.puts "File.open('a', 'w') {}" + f.puts "File.open('b', 'w') {}" + f.puts "STDOUT.puts [11,22,33]" + f.puts "STDERR.puts [44,55,66]" + f.puts "exit 39" +end +cmd "ruby script.rb" do +end + +# stdout begin +# ### ruby script.rb +# --- ERROR: expected zero exit status, got 39 +# --- ERROR: wrong stdout +# --- actual: 11 +# --- 22 +# --- 33 +# --- expect: [[empty]] +# --- ERROR: wrong stderr +# --- actual: 44 +# --- 55 +# --- 66 +# --- expect: [[empty]] +# --- ERROR: created files +# --- actual: ["a", "b"] +# --- expect: [] +# --- ERROR: removed files +# --- actual: ["before1", "before2"] +# --- expect: [] +# stdout end + +#----------------------------------- +# removed_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "rm.rb file1" do + comment "removed_files" + removed_files "file1" +end + +# stdout begin +# ### removed_files +# stdout end + +#----------------------------------- +# FAILED removed_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "true.rb" do + comment "FAILED removed_files" + removed_files "file1" +end + +# stdout begin +# ### FAILED removed_files +# --- ERROR: removed files +# --- actual: [] +# --- expect: ["file1"] +# stdout end + +#----------------------------------- +# changed_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "sleep.rb 1 && touch.rb file1" do + comment "changed_files" + changed_files "file1" +end + +# stdout begin +# ### changed_files +# stdout end + +#----------------------------------- +# FAILED changed_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "true.rb" do + comment "FAILED changed_files" + changed_files "file1" +end + +# stdout begin +# ### FAILED changed_files +# --- ERROR: changed files +# --- actual: [] +# --- expect: ["file1"] +# stdout end + +#----------------------------------- +# created_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "touch.rb file3" do + comment "created_files" + created_files "file3" +end + +# stdout begin +# ### created_files +# stdout end + +#----------------------------------- +# FAILED created_files + +File.open("file1", "w") {} +File.open("file2", "w") {} + +cmd "true.rb" do + comment "FAILED created_files" + created_files "file3" +end + +# stdout begin +# ### FAILED created_files +# --- ERROR: created files +# --- actual: [] +# --- expect: ["file3"] +# stdout end + +#----------------------------------- +# with comment + +cmd "true.rb" do + comment "this-is-the-comment" +end + +# stdout begin +# ### this-is-the-comment +# stdout end + +#----------------------------------- +# exit_nonzero + +cmd "exit.rb 33" do + comment "exit_nonzero" + exit_nonzero +end + +# stdout begin +# ### exit_nonzero +# stdout end + +#----------------------------------- +# FAILING exit_nonzero + +cmd "exit.rb 0" do + comment "failing exit_nonzero" + exit_nonzero +end + +# stdout begin +# ### failing exit_nonzero +# --- ERROR: expected nonzero exit status +# stdout end + +#----------------------------------- +# exit_status + +cmd "exit.rb 33" do + comment "exit_status" + exit_status 33 +end + +# stdout begin +# ### exit_status +# stdout end + +#----------------------------------- +# FAILING exit_status + +cmd "exit.rb 44" do + comment "failing exit_status" + exit_status 33 +end + +# stdout begin +# ### failing exit_status +# --- ERROR: expected 33 exit status, got 44 +# stdout end + +#----------------------------------- +# stdout_equal -- one line + +cmd "lines.rb 11" do + comment "stdout_equal" + stdout_equal "11\n" +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# FAILING stdout_equal -- one line + +cmd "lines.rb 22" do + comment "stdout_equal" + stdout_equal "11\n" +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 22 +# --- expect: 11 +# stdout end + +#----------------------------------- +# stdout_equal -- two lines + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal "11\n22\n" +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# FAILING stdout_equal -- two lines + +cmd "lines.rb 33 44" do + comment "stdout_equal" + stdout_equal "11\n22\n" +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 33 +# --- 44 +# --- expect: 11 +# --- 22 +# stdout end + +#----------------------------------- +# stdout_equal(arr) -- two lines + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal ["11", "22"] +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# FAILING stdout_equal(arr) -- two lines + +cmd "lines.rb 33 44" do + comment "stdout_equal" + stdout_equal ["11", "22"] +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 33 +# --- 44 +# --- expect: 11 +# --- 22 +# stdout end + +#----------------------------------- +# FAILING stdout_equal(arr) -- different # lines + +cmd "lines.rb 11 22 33" do + comment "stdout_equal" + stdout_equal ["11", "22"] +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 11 +# --- 22 +# --- 33 +# --- expect: 11 +# --- 22 +# stdout end + +#----------------------------------- +# stdout_equal -- regexp argument + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal /^22$/ +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# stdout_equal -- twice, regexp argument + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal /^22$/ + stdout_equal /^11$/ +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# FAILING first, stdout_equal -- twice, regexp argument + +cmd "lines.rb 99 22" do + comment "stdout_equal" + stdout_equal /^22$/ + stdout_equal /^11$/ +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 99 +# --- 22 +# --- expect: (?-mix:^11$) +# stdout end + +#----------------------------------- +# FAILING second, stdout_equal -- twice, regexp argument + +cmd "lines.rb 11 99" do + comment "stdout_equal" + stdout_equal /^22$/ + stdout_equal /^11$/ +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 11 +# --- 99 +# --- expect: (?-mix:^22$) +# stdout end + +#----------------------------------- +# FAILING stdout_equal -- regexp argument + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal /^\d+ \d+$/ +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 11 +# --- 22 +# --- expect: (?-mix:^\d+ \d+$) +# stdout end + +#----------------------------------- +# stdout_equal(arr) -- regexp argument + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal ["11", /^22$/] +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# stdout_equal(arr) -- regexp argument (II) + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal ["11", /^\d+$/] +end + +# stdout begin +# ### stdout_equal +# stdout end + +#----------------------------------- +# FAILING stdout_equal(arr) -- regexp argument + +cmd "lines.rb 11 22" do + comment "stdout_equal" + stdout_equal ["11", /^\d+ \d+$/] +end + +# stdout begin +# ### stdout_equal +# --- ERROR: wrong stdout +# --- actual: 11 +# --- 22 +# --- expect: 11 +# --- (?-mix:^\d+ \d+$) +# stdout end + +#====================================================================== + +#----------------------------------- +# symlinks in tree -- should work +# TODO: this test should be improved to actually trigger the difference +# between lstat/stat in "_update_hardlinks". +# SKIP mswin32 + +File.symlink "non-existing", "non-existing-link" + +File.open("existing", "w") {} +File.symlink "existing", "existing-link" + +cmd "true.rb" do +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# file with mtime in future + +File.open("future-file", "w") {} +future = Time.now + 86400 +File.utime future, future, "future-file" + +cmd "true.rb" do +end + +# stdout begin +# ### true.rb +# stdout end + diff --git a/t/01-simple.rb b/t/01-simple.rb new file mode 100644 index 0000000..4749776 --- /dev/null +++ b/t/01-simple.rb @@ -0,0 +1,341 @@ + +#----------------------------------- +# try to run non-existing command + +cmd "non-existing" do + exit_nonzero + stderr_equal /non-existing: .*not found/ +end + +# stdout begin +# ### non-existing +# stdout end + +#----------------------------------- +# FAILING try to run non-existing command + +cmd "non-existing" do +end + +# stdout begin +# ### non-existing +# --- ERROR: expected zero exit status, got 127 +# --- ERROR: wrong stderr +#/--- actual:.*non-existing: .*not found +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# "true.rb" is archetypic command: zero exit status, no output + +cmd "true.rb" do +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - explicit exit_zero + +cmd "true.rb" do + exit_zero +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect exit_nonzero + +cmd "true.rb" do + exit_nonzero +end + +# stdout begin +# ### true.rb +# --- ERROR: expected nonzero exit status +# stdout end + +#----------------------------------- +# true - incorrect exit_status + +cmd "true.rb" do + exit_status 18 +end + +# stdout begin +# ### true.rb +# --- ERROR: expected 18 exit status, got 0 +# stdout end + +#----------------------------------- +# true - correct exit_status + +cmd "true.rb" do + exit_status 0 +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect stdout + +cmd "true.rb" do + stdout_equal ["hello"] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stdout +# --- actual: [[empty]] +# --- expect: hello +# stdout end + +#----------------------------------- +# true - correct stdout + +cmd "true.rb" do + stdout_equal [] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect stderr + +cmd "true.rb" do + stderr_equal ["hello"] +end + +# stdout begin +# ### true.rb +# --- ERROR: wrong stderr +# --- actual: [[empty]] +# --- expect: hello +# stdout end + +#----------------------------------- +# true - correct stderr + +cmd "true.rb" do + stderr_equal [] +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect created_files + +cmd "true.rb" do + created_files "foo" +end + +# stdout begin +# ### true.rb +# --- ERROR: created files +# --- actual: [] +# --- expect: ["foo"] +# stdout end + +#----------------------------------- +# true - incorrect created_files + +cmd "true.rb" do + created_files "foo", "bar" +end + +# stdout begin +# ### true.rb +# --- ERROR: created files +# --- actual: [] +# --- expect: ["bar", "foo"] +# stdout end + +#----------------------------------- +# true - correct created_files + +cmd "true.rb" do + created_files +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect changed_files + +cmd "true.rb" do + changed_files "foo" +end + +# stdout begin +# ### true.rb +# --- ERROR: changed files +# --- actual: [] +# --- expect: ["foo"] +# stdout end + +#----------------------------------- +# true - incorrect changed_files + +cmd "true.rb" do + changed_files "foo", "bar" +end + +# stdout begin +# ### true.rb +# --- ERROR: changed files +# --- actual: [] +# --- expect: ["bar", "foo"] +# stdout end + +#----------------------------------- +# true - correct changed_files + +cmd "true.rb" do + changed_files +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# true - incorrect removed_files + +cmd "true.rb" do + removed_files "foo" +end + +# stdout begin +# ### true.rb +# --- ERROR: removed files +# --- actual: [] +# --- expect: ["foo"] +# stdout end + +#----------------------------------- +# true - incorrect removed_files + +cmd "true.rb" do + removed_files "foo", "bar" +end + +# stdout begin +# ### true.rb +# --- ERROR: removed files +# --- actual: [] +# --- expect: ["bar", "foo"] +# stdout end + +#----------------------------------- +# true - correct removed_files + +cmd "true.rb" do + removed_files +end + +# stdout begin +# ### true.rb +# stdout end + +#====================================================================== +# test - without assertions + +#----------------------------------- +# without assertions -- correct + +cmd "true.rb" do +end + +# stdout begin +# ### true.rb +# stdout end + +#----------------------------------- +# without assertions -- incorrect exit status + +cmd "false.rb" do +end + +# stdout begin +# ### false.rb +# --- ERROR: expected zero exit status, got 1 +# stdout end + +#----------------------------------- +# without assertions -- incorrect stdout + +cmd "echo.rb hello" do +end + +# stdout begin +# ### echo.rb hello +# --- ERROR: wrong stdout +# --- actual: hello +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# without assertions -- incorrect stderr + +cmd "echo.rb hello >&2" do +end + +# stdout begin +# ### echo.rb hello >&2 +# --- ERROR: wrong stderr +# --- actual: hello +# --- expect: [[empty]] +# stdout end + +#----------------------------------- +# without assertions -- incorrect created_files + +cmd "touch.rb new_file" do +end + +# stdout begin +# ### touch.rb new_file +# --- ERROR: created files +# --- actual: ["new_file"] +# --- expect: [] +# stdout end + +#----------------------------------- +# without assertions -- incorrect changed_files + +touch_file "changed_file" +cmd "echo.rb ... >> changed_file" do +end + +# stdout begin +# ### echo.rb ... >> changed_file +# --- ERROR: changed files +# --- actual: ["changed_file"] +# --- expect: [] +# stdout end + +#----------------------------------- +# without assertions -- incorrect removed_files + +touch_file "removed_file" +cmd "rm.rb removed_file" do +end + +# stdout begin +# ### rm.rb removed_file +# --- ERROR: removed files +# --- actual: ["removed_file"] +# --- expect: [] +# stdout end +