Merge holmberg556/cmdtest into master

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+tmp-cmdtest-1
+tmp-cmdtest-python

CMDTEST_example.yml

@@ -0,0 +1,11 @@
+
+
+- test: simple
+  actions:
+    - create_file:
+        name: foo.txt
+        content: "abc ..."
+    - cmd:
+        cmdline: echo hello
+        stdout: /hello/
+        status: 0

CMakeLists.txt

@@ -0,0 +1,92 @@
+cmake_minimum_required(VERSION 3.0)
+project(cmdtest)
+
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ruby bin/cmdtest.rb --shortversion
+  OUTPUT_VARIABLE CMDTEST_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND date +%Y%m%d
+  OUTPUT_VARIABLE CMDTEST_DATE
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+set(CPACK_GENERATOR "STGZ;TGZ;TZ")
+if( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" )
+  set(CPACK_GENERATOR "DEB;${CPACK_GENERATOR}")
+endif()
+
+set(CPACK_PACKAGE_VERSION "${CMDTEST_VERSION}.${CMDTEST_DATE}")
+
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Johan Holmberg <holmberg556@gmail.com>")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Cmdtest, xUnit style testing of commands")
+set(CPACK_PACKAGE_DESCRIPTION "Cmdtest, xUnit style testing of commands ...")
+
+set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "all")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "ruby | ruby-interpreter")
+
+INCLUDE(CPack)
+
+execute_process(
+  COMMAND git rev-parse --short HEAD
+  OUTPUT_VARIABLE GIT_REV
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+execute_process(
+  COMMAND git show -s --format=%ci HEAD
+  OUTPUT_VARIABLE GIT_DATE
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+message(STATUS "GIT_REV  = '${GIT_REV}'")
+message(STATUS "GIT_DATE = '${GIT_DATE}'")
+
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ./replace_strings.pl
+  "GIT_REV_STRING=${GIT_REV}"
+  "GIT_DATE_STRING=${GIT_DATE}"
+  "VERSION=${CPACK_PACKAGE_VERSION}"
+  bin/cmdtest.rb bin/cmdtest.rb.generated
+  )
+
+
+install(
+  FILES
+  lib/cmdtest/argumentparser.rb
+  lib/cmdtest/baselogger.rb
+  lib/cmdtest/cmdeffects.rb
+  lib/cmdtest/consolelogger.rb
+  lib/cmdtest/fileinfo.rb
+  lib/cmdtest/fssnapshot.rb
+  lib/cmdtest/junitfile.rb
+  lib/cmdtest/junitlogger.rb
+  lib/cmdtest/lcs.rb
+  lib/cmdtest/methodfilter.rb
+  lib/cmdtest/notify.rb
+  lib/cmdtest/output.rb
+  lib/cmdtest/testcase.rb
+  lib/cmdtest/util.rb
+  lib/cmdtest/workdir.rb
+  DESTINATION lib/ruby/vendor_ruby/cmdtest
+  )
+
+install(
+  FILES
+  doc/cmdtest.html
+  COPYING.txt
+  README.html
+  DESTINATION share/doc/rake
+  )
+
+install(
+  PROGRAMS
+  bin/cmdtest.rb.generated
+  DESTINATION bin
+  RENAME cmdtest
+  )

Rakefile

@@ -17,3 +17,9 @@ task "readme-html" do
   sh "rst2html.py -gds --stylesheet doc/rst.css README.rst README.html"
 end
 
+desc "generate DEB pacakge"
+task "generate-debian-package" do
+  sh "rm -rf build"
+  sh "mkdir build"
+  sh "cd build && cmake .. && make package"
+end

bin/cmdtest.rb

@@ -2,7 +2,7 @@
 #----------------------------------------------------------------------
 # cmdtest.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -25,9 +25,9 @@
 # found in the files. The result can be reported in different ways.
 # Most of the testing logic is found in the library files "cmdtest/*.rb".
 
-top_dir = File.dirname(File.dirname(__FILE__))
+TOP_DIR = File.expand_path(File.dirname(File.dirname(__FILE__)))
-lib_dir = File.join(File.expand_path(top_dir), "lib")
+LIB_DIR = File.join(TOP_DIR, "lib")
-$:.unshift(lib_dir) if File.directory?(File.join(lib_dir, "cmdtest"))
+$:.unshift(LIB_DIR) if File.directory?(File.join(LIB_DIR, "cmdtest"))
 
 require "cmdtest/argumentparser"
 require "cmdtest/baselogger"
@@ -47,6 +47,14 @@ require "stringio"
 
 module Cmdtest
 
+  ORIG_CWD = Dir.pwd
+
+  GIT_REV  = '$GIT_REV_STRING$'
+  GIT_DATE = '$GIT_DATE_STRING$'
+  VERSION  = '$VERSION$'
+
+  SHORT_VERSION = '1.4'
+
   #----------------------------------------------------------------------
 
   module LogBaseMixin
@@ -164,18 +172,22 @@ module Cmdtest
     end
 
     def run(clog, runner)
+      ok = false
       clog.notify("testmethod", @method) do
         obj = @adm_class.runtime_class.new(self, clog, runner)
-        if runner.opts.parallel == 1
+        Dir.chdir(obj._work_dir.path)
-          Dir.chdir(obj._work_dir.path)
-        end
-        obj.setup
         begin
+          obj.setup
           obj.send(@method)
+          Dir.chdir(obj._work_dir.path)
+          obj.teardown
+
           clog.assert_success
           runner.method_filter.success(method_id)
+          ok = true
         rescue Cmdtest::AssertFailed => e
           clog.assert_failure(e.message)
+          runner.method_filter.failure(method_id)
         rescue => e
           io = StringIO.new
           io.puts "CAUGHT EXCEPTION:"
@@ -183,9 +195,12 @@ module Cmdtest
           io.puts "BACKTRACE:"
           io.puts e.backtrace.map {|line| "  " + line }
           clog.assert_error(io.string)
+          runner.method_filter.failure(method_id)
         end
-        obj.teardown
       end
+      return ok
+    ensure
+      Dir.chdir(ORIG_CWD)
     end
 
   end
@@ -200,7 +215,7 @@ module Cmdtest
       @runtime_class, @adm_file, @runner = runtime_class, adm_file, runner
 
       tested = runner.opts.test
-      @adm_methods = @runtime_class.public_instance_methods(false).select do |name|
+      @adm_methods = @runtime_class.public_instance_methods(false).sort.select do |name|
         name =~ /^test_/
       end.map do |name|
         TestMethod.new(name, self, runner)
@@ -248,8 +263,6 @@ module Cmdtest
 
     attr_reader :opts, :orig_cwd, :method_filter
 
-    ORIG_CWD = Dir.pwd
-
     def initialize(project_dir, incremental, opts)
       @project_dir = project_dir
       @opts = opts
@@ -424,7 +437,11 @@ module Cmdtest
             for adm_class in adm_file.adm_classes
               clog.notify("testclass", adm_class.runtime_class.display_name) do
                 for adm_method in adm_class.adm_methods
-                  adm_method.run(clog, self)
+                  ok = adm_method.run(clog, self)
+                  if !ok && @opts.stop_on_error
+                    puts "cmdtest: exiting after first error ..."
+                    exit(1)
+                  end
                   if $cmdtest_got_ctrl_c > 0
                     puts "cmdtest: exiting after Ctrl-C ..."
                     exit(1)
@@ -440,16 +457,14 @@ module Cmdtest
 
     def report_result(error_logger)
       if ! opts.quiet
-        puts
+        summary = {
-        puts "%s %d test classes, %d test methods, %d commands, %d errors, %d fatals." % [
+          "classes"  => error_logger.n_classes,
-          error_logger.n_failures == 0 && error_logger.n_errors == 0 ? "###" : "---",
+          "methods"  => error_logger.n_methods,
-          error_logger.n_classes,
+          "commands" => error_logger.n_commands,
-          error_logger.n_methods,
+          "failures" => error_logger.n_failures,
-          error_logger.n_commands,
+          "errors"   => error_logger.n_errors,
-          error_logger.n_failures,
+        }
-          error_logger.n_errors,
+        Cmdtest.print_summary(summary)
-        ]
-        puts
       end
 
       ok = error_logger.everything_ok?
@@ -461,7 +476,15 @@ module Cmdtest
 
   #----------------------------------------------------------------------
 
+  START_TIME = Time.now
+
   def self.print_summary(summary)
+    s = (Time.now - START_TIME).to_i
+    m, s = s.divmod(60)
+    h, m = m.divmod(60)
+
+    puts "###"
+    puts "### Finished: %s,  Elapsed: %02d:%02d:%02d" % [Time.now.strftime("%F %T"), h,m,s]
     puts
     puts "%s %d test classes, %d test methods, %d commands, %d errors, %d fatals." % [
       summary["failures"] == 0 && summary["errors"] == 0 ? "###" : "---",
@@ -478,8 +501,6 @@ module Cmdtest
 
   class ProjectDir
 
-    ORIG_CWD = Dir.pwd
-
     def initialize(argv)
       @argv = argv
       @test_filenames = nil
@@ -533,7 +554,7 @@ module Cmdtest
             exit(1)
           end
         elsif File.directory?(arg)
-          Dir.foreach(arg) do |entry|
+          for entry in Dir.entries(arg).sort
             path = File.join(arg,entry)
             next unless File.file?(path)
             next unless entry =~ /^CMDTEST_.*\.rb$/
@@ -557,23 +578,57 @@ module Cmdtest
 
     def _parse_options
       pr = @argument_parser = ArgumentParser.new("cmdtest")
+      pr.add("-h", "--help",         "show this help message and exit")
+      pr.add("",   "--shortversion", "show just version number")
       pr.add("",   "--version",      "show version")
       pr.add("-q", "--quiet",        "be more quiet")
       pr.add("-v", "--verbose",      "be more verbose")
+      pr.add("",   "--diff",         "experimental diff output")
       pr.add("",   "--fast",         "run fast without waiting for unique mtime:s")
       pr.add("-j", "--parallel",     "build in parallel",  type: Integer, default: 1, metavar: "N")
       pr.add("",   "--test",         "only run named test", type: [String])
       pr.add("",   "--xml",          "write summary on JUnit format", type: String, metavar: "FILE")
       pr.add("",   "--no-exit-code", "exit with 0 status even after errors")
+      pr.add("",   "--stop-on-error","exit after first error")
       pr.add("-i", "--incremental",  "incremental mode")
       pr.add("",   "--slave",        "run in slave mode", type: String)
       pr.addpos("arg", "testfile or pattern", nargs: 0..999)
-      return pr.parse_args(ARGV, patterns: [], ruby_s: Util.windows?)
+
+      opts = pr.parse_args(ARGV, patterns: [], ruby_s: Util.windows?)
+      if opts.help
+        pr.print_usage()
+        exit(0)
+      end
+      return opts
     end
 
     def run
       opts = _parse_options
 
+      if opts.shortversion
+        puts SHORT_VERSION
+        exit(0)
+      elsif opts.version
+        puts "Version:   " + (VERSION =~ /^\$/ ? SHORT_VERSION : VERSION)
+        if File.directory?(File.join(TOP_DIR, ".git"))
+          Dir.chdir(TOP_DIR) do
+            git_rev = `git rev-parse HEAD`
+            git_date = `git show -s --format=%ci HEAD`
+            puts "Revision:  #{git_rev}"
+            puts "Date:      #{git_date}"
+          end
+        else
+          puts "Revision:  #{GIT_REV}"
+          puts "Date:      #{GIT_DATE}"
+        end
+        exit(0)
+      end
+
+      if opts.stop_on_error && opts.parallel != 1
+        puts "cmdtest: error: --stop-on-error can not be used with --parallel"
+        exit(1)
+      end
+
       _update_cmdtest_level(opts.slave ? 0 : 1)
 
       files = []

bin/test_argparse.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+
+top_dir = File.dirname(File.dirname(__FILE__))
+lib_dir = File.join(File.expand_path(top_dir), "lib")
+$:.unshift(lib_dir) if File.directory?(File.join(lib_dir, "cmdtest"))
+
+require "cmdtest/argumentparser"
+
+
+pr = Cmdtest::ArgumentParser.new("jcons_cmds")
+
+pr.add("-q", "--quiet",       "be more quiet")
+pr.add("-j", "--parallel",    "build in parallel",  type: Integer, default: 1, metavar: "N")
+
+pr.add("-k", "--keep-going",  "continue after errors")
+pr.add("-B", "--always-make", "always build targets")
+pr.add("-r", "--remove",      "remove targets")
+pr.add("",   "--accept-existing-target", "make updating an existing target a nop")
+
+pr.add("", "--dont-trust-mtime", "always consult files for content digest")
+
+pr.add("-f", "--file",        "name of *.cmds file", type: [String])
+pr.add("-v", "--verbose",     "be more verbose")
+pr.add("",   "--version",     "show version")
+
+pr.add("",   "--cache-dir",   "name of cache directory", type: String)
+pr.add("",   "--cache-force", "copy existing files into cache")
+
+pr.add("-p", "--list-targets", "list known targets")
+pr.add("",   "--list-commands","list known commands")
+pr.add("",   "--log-states",   "log state machine")
+
+pr.addpos("arg", "target or NAME=VALUE", nargs: 0..999)
+
+opts = pr.parse_args(ARGV)
+
+p opts

cmdtest.gemspec

@@ -0,0 +1,30 @@
+
+Gem::Specification.new do |s|
+    s.name        = 'cmdtest'
+    s.version     = '0.1.0'
+    s.date        = '2015-09-21'
+    s.summary     = "Cmdtest, xUnit style test fo commands"
+    s.description = "???"
+    s.authors     = ["Johan Holmberg"]
+    s.email       = 'holmberg556@gmail.com'
+    s.files       = [
+        'lib/cmdtest/argumentparser.rb',
+        'lib/cmdtest/baselogger.rb',
+        'lib/cmdtest/cmdeffects.rb',
+        'lib/cmdtest/consolelogger.rb',
+        'lib/cmdtest/fileinfo.rb',
+        'lib/cmdtest/fssnapshot.rb',
+        'lib/cmdtest/junitfile.rb',
+        'lib/cmdtest/junitlogger.rb',
+        'lib/cmdtest/methodfilter.rb',
+        'lib/cmdtest/notify.rb',
+        'lib/cmdtest/output.rb',
+        'lib/cmdtest/testcase.rb',
+        'lib/cmdtest/util.rb',
+        'lib/cmdtest/workdir.rb',
+    ]
+    s.executables << "cmdtest.rb"
+    s.homepage =
+        'https://bitbucket.org/holmberg556/cmdtest'
+    s.license = 'GNU'
+end

doc/CMDTEST_example.rb

@@ -0,0 +1,20 @@
+class CMDTEST_example < Cmdtest::Testcase
+
+  def test_hello_world
+    cmd "echo hello" do
+      stdout_equal "hello\n"
+    end
+
+    cmd "echo WORLD" do
+      stdout_equal "world\n"
+    end
+  end
+
+  def test_touch_and_exit
+    cmd "touch bar.txt ; exit 8" do
+      created_files "foo.txt"
+      exit_status 7
+    end
+  end
+
+end

doc/cmdtest.html

@@ -497,7 +497,7 @@ end
 ...
 </pre>
 <p>The list of such helper functions includes:
-<tt class="docutils literal">create_file</tt>, <tt class="docutils literal">touch_file</tt>, <tt class="docutils literal">import_file</tt> and <tt class="docutils literal">ignore_file</tt>.
+<tt class="docutils literal">create_file</tt>, <tt class="docutils literal">touch_file</tt>, <tt class="docutils literal">import_file</tt> , <tt class="docutils literal">import_directory</tt> and <tt class="docutils literal">ignore_file</tt>.
 Beside these methods the test can of course also contain arbitrary Ruby-code.</p>
 </div>
 <div class="section" id="work-directory">
@@ -807,6 +807,12 @@ The <tt class="docutils literal">src</tt> path is evaluated relative to the curr
 when <tt class="docutils literal">cmdtest</tt> was called. The <tt class="docutils literal">tgt</tt> is evaluated relative to
 the current directory inside the &quot;work directory&quot; at the time
 of the call.</dd>
+<dt><tt class="docutils literal">import_directory(src, tgt)</tt></dt>
+<dd>Copy a directory tree from outside of the &quot;work directory&quot; to inside.
+The <tt class="docutils literal">src</tt> path is evaluated relative to the current directory
+when <tt class="docutils literal">cmdtest</tt> was called. The <tt class="docutils literal">tgt</tt> is evaluated relative to
+the current directory inside the &quot;work directory&quot; at the time
+of the call. It is an error if <tt class="docutils literal">tgt</tt> already exists.</dd>
 <dt><tt class="docutils literal">prepend_local_path(dir)</tt></dt>
 <dd>Prepend the given directory to the <tt class="docutils literal">PATH</tt> so commands executed via <tt class="docutils literal">cmd</tt>
 are looked up using the modified <tt class="docutils literal">PATH</tt>. The argument <tt class="docutils literal">dir</tt> is evaluated
@@ -859,7 +865,7 @@ time of the call.</dd>
 <div class="footer">
 <hr class="footer" />
 <a class="reference external" href="cmdtest.txt">View document source</a>.
-Generated on: 2015-08-26.
+Generated on: 2016-04-14.
 Generated by <a class="reference external" href="http://docutils.sourceforge.net/">Docutils</a> from <a class="reference external" href="http://docutils.sourceforge.net/rst.html">reStructuredText</a> source.
 
 </div>

doc/cmdtest.txt

@@ -190,7 +190,7 @@ the creation of files::
   ...
 
 The list of such helper functions includes:
-``create_file``, ``touch_file``, ``import_file`` and ``ignore_file``.
+``create_file``, ``touch_file``, ``import_file`` , ``import_directory`` and ``ignore_file``.
 Beside these methods the test can of course also contain arbitrary Ruby-code.
 
 
@@ -537,6 +537,13 @@ or in the ``setup`` method.
     the current directory inside the "work directory" at the time
     of the call.
 
+``import_directory(src, tgt)``
+    Copy a directory tree from outside of the "work directory" to inside.
+    The ``src`` path is evaluated relative to the current directory
+    when ``cmdtest`` was called. The ``tgt`` is evaluated relative to
+    the current directory inside the "work directory" at the time
+    of the call. It is an error if ``tgt`` already exists.
+
 ``prepend_local_path(dir)``
     Prepend the given directory to the ``PATH`` so commands executed via ``cmd``
     are looked up using the modified ``PATH``. The argument ``dir`` is evaluated

examples/CMDTEST_gcc.rb

@@ -38,9 +38,7 @@ class CMDTEST_gcc < Cmdtest::Testcase
   def test_no_arguments
     cmd "#{gcc}" do
       comment "no arguments"
-      stderr_equal [
+      stderr_equal /^.*gcc.*: no input files/
-        /^.*gcc.*: no input files/,
-      ]
       exit_nonzero
     end
   end
@@ -49,26 +47,20 @@ class CMDTEST_gcc < Cmdtest::Testcase
 
   def test_nonexisting_input
     cmd "#{gcc} -c non-existing.c" do
-      stderr_equal [
+      stderr_equal /^.*gcc.*: non-existing.c: No such file/
-        /^.*gcc.*: non-existing.c: No such file/,
+      stderr_equal /^.*gcc.*: no input files/
-        /^.*gcc.*: no input files/,
-      ]
       exit_nonzero
     end
 
     cmd "#{gcc} non-existing.c" do
-      stderr_equal [
+      stderr_equal /^.*gcc.*: non-existing.c: No such file/
-        /^.*gcc.*: non-existing.c: No such file/,
+      stderr_equal /^.*gcc.*: no input files/
-        /^.*gcc.*: no input files/,
-      ]
       exit_nonzero
     end
 
     cmd "#{gcc} non-existing.o" do
-      stderr_equal [
+      stderr_equal /^.*gcc.*: non-existing.o: No such file/
-        /^.*gcc.*: non-existing.o: No such file/,
+      stderr_equal /^.*gcc.*: no input files/
-        /^.*gcc.*: no input files/,
-      ]
       exit_nonzero
     end
   end
@@ -165,7 +157,7 @@ class CMDTEST_gcc < Cmdtest::Testcase
 
     cmd "#{gcc} -o gamma.o -c alpha.c beta.c" do
       comment "-o with two source files give error"
-      stderr_equal /^.*gcc.*: cannot specify -o with -c or -S with multiple files/
+      stderr_equal /^.*gcc.*: cannot specify -o with -c.* with multiple files/
       exit_nonzero
     end
   end

files/bin/hello1

@@ -0,0 +1,3 @@
+#!/usr/bin/perl
+use File::Basename;
+print basename($0),"\n";

lib/cmdtest/argumentparser.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # argumentparser.rb
 #----------------------------------------------------------------------
-# Copyright 2015 Johan Holmberg.
+# Copyright 2015-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -133,8 +133,6 @@ module Cmdtest
             @help = false
 
             @args = []
-
-            add("-h", "--help", "show this help message and exit")
         end
 
         def add(sname, name, help, args = {})
@@ -182,7 +180,6 @@ module Cmdtest
                 puts
             end
             puts("optional arguments:")
-            puts("  -h, --help            show this help message and exit")
             for option in @options
                 str = "  " + option.names()
                 wanted = 22
@@ -207,11 +204,6 @@ module Cmdtest
 
             @optind = 0
             while _more_args() && _arg() =~ /^-./
-                if _arg() == "-h"
-                    print_usage()
-                    exit(0)
-                end
-
                 if _arg() == "--"
                     @optind += 1
                     break

lib/cmdtest/baselogger.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # baselogger.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/cmdeffects.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # cmdeffects.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/consolelogger.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # consolelogger.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/fileinfo.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # fileinfo.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/fssnapshot.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # fssnapshot.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/junitfile.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # junitfile.rb
 #----------------------------------------------------------------------
-# Copyright 2009-2014 Johan Holmberg.
+# Copyright 2009-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/junitlogger.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # junitlogger.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/lcs.rb

@@ -0,0 +1,134 @@
+#----------------------------------------------------------------------
+# lcs.rb
+#----------------------------------------------------------------------
+# Copyright 2016 Johan Holmberg.
+#----------------------------------------------------------------------
+# This file is part of "cmdtest".
+#
+# "cmdtest" is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# "cmdtest" is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with "cmdtest".  If not, see <http://www.gnu.org/licenses/>.
+#----------------------------------------------------------------------
+
+module Cmdtest
+
+  class LCS
+    def print()
+      i = @m
+      j = @n
+      while i > 0 || j > 0
+        if @b[i][j] == :XY
+          acc = []
+          while @b[i][j] == :XY
+            acc << @y[j]
+            i -= 1
+            j -= 1
+          end
+          found_xy(acc)
+        elsif i > 0 && (j == 0 || @b[i][j] == :X)
+          acc = []
+          while i > 0 && (j == 0 || @b[i][j] == :X)
+            acc << @x[i]
+            i -= 1
+          end
+          found_x(acc)
+        elsif j > 0 && (i == 0 || @b[i][j] == :Y)
+          acc = []
+          while j > 0 && (i == 0 || @b[i][j] == :Y)
+            acc << @y[j]
+            j -= 1
+          end
+          found_y(acc)
+        else
+          raise "internal error"
+        end
+      end
+    end
+
+    def initialize(x, y)
+      @m = x.size
+      @n = y.size
+      @x = [nil] + x.reverse
+      @y = [nil] + y.reverse
+      @c = Array.new(@m+1) { Array.new(@n+1) }
+      @b = Array.new(@m+1) { Array.new(@n+1) }
+
+      for i in 1..@m
+        @c[i][0] = 0
+      end
+
+      for j in 0..@n
+        @c[0][j] = 0
+      end
+
+      for i in 1..@m
+        for j in 1..@n
+          if @x[i] === @y[j]
+            @c[i][j] = @c[i-1][j-1] + 1
+            @b[i][j] = :XY
+          elsif @c[i-1][j] >= @c[i][j-1]
+            @c[i][j] = @c[i-1][j]
+            @b[i][j] = :X
+          else
+            @c[i][j] = @c[i][j-1]
+            @b[i][j] = :Y
+          end
+        end
+      end
+    end
+
+    def found_xy(arr)
+    end
+
+    def found_x(arr)
+    end
+
+    def found_y(arr)
+    end
+  end
+
+  class DiffLCS < LCS
+    include Enumerable
+
+    def found_x(arr)
+      for e in arr
+        if Regexp === e
+          @arr << "- " + e.inspect
+        else
+          @arr << "- " + e
+        end
+      end
+    end
+
+    def found_y(arr)
+      for e in arr
+        @arr << "+ " + e
+      end
+    end
+
+    def found_xy(arr)
+      for e in arr
+        @arr << "  " + e
+      end
+    end
+
+    def each
+      @arr = []
+      print
+      for e in @arr
+        yield e
+      end
+    end
+  end
+
+
+end

lib/cmdtest/methodfilter.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # methodfilter.rb
 #----------------------------------------------------------------------
-# Copyright 2009-2014 Johan Holmberg.
+# Copyright 2009-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -63,6 +63,10 @@ module Cmdtest
       @new_filter[method_id.key] = _get_method_signature(method_id)
     end
 
+    def failure(method_id)
+      @new_filter.delete(method_id.key)
+    end
+
     def _maybe_read_ruby_file(file)
       return if @files_read[file]
       @files_read[file] = true

lib/cmdtest/notify.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # notify.rb
 #----------------------------------------------------------------------
-# Copyright 2012-2014 Johan Holmberg.
+# Copyright 2012-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/output.rb

@@ -2,7 +2,7 @@
 #----------------------------------------------------------------------
 # output.rb
 #----------------------------------------------------------------------
-# Copyright 2010-2014 Johan Holmberg.
+# Copyright 2010-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #

lib/cmdtest/testcase.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # testcase.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -23,9 +23,12 @@ require "fileutils"
 require "set"
 require "stringio"
 
+require "cmdtest/lcs"
+
 module Cmdtest
 
   class AssertFailed < RuntimeError ; end
+  class UsageError < RuntimeError ; end
 
   # Base class for testcases.
   # Some attributes and methods are prefixed with an "_" to avoid
@@ -63,7 +66,7 @@ module Cmdtest
 
     ORIG_CWD = Dir.pwd
 
-    attr_reader :_work_dir, :_env
+    attr_reader :_work_dir, :_env_setenv
 
     def initialize(test_method, clog, runner)
       @_test_method = test_method
@@ -71,7 +74,7 @@ module Cmdtest
       @_runner = runner
       @_work_dir = Workdir.new(self, runner)
       @_cwd = @_runner.tmp_work_dir
-      @_env = Hash.new
+      @_env_setenv = Hash.new
       @_in_cmd = false
       @_comment_str = nil
       @_env_path = @_runner.orig_env_path
@@ -88,7 +91,30 @@ module Cmdtest
       src_path = File.expand_path(src, @_runner.test_files_top)
       tgt_path = _cwd_path(tgt)
       FileUtils.mkdir_p(File.dirname(tgt_path))
-      FileUtils.cp(src_path, tgt_path)
+      if File.file?(src_path)
+        FileUtils.cp(src_path, tgt_path)
+      else
+        raise UsageError, "'import_file' argument not a file: '#{src}'"
+      end
+    end
+
+    #------------------------------
+    # Import directory into the "workdir" from the outside world.
+    # The source is found relative to the current directory when "cmdtest"
+    # was invoked. The target is created inside the "workdir" relative to
+    # the current directory at the time of the call.
+
+    def import_directory(src, tgt)
+      src_path = File.expand_path(src, @_runner.test_files_top)
+      tgt_path = _cwd_path(tgt)
+      FileUtils.mkdir_p(File.dirname(tgt_path))
+      if File.exists?(tgt_path)
+        raise UsageError, "'import_directory' target argument already exist: '#{tgt}'"
+      elsif File.directory?(src_path)
+        FileUtils.cp_r(src_path, tgt_path)
+      else
+        raise UsageError, "'import_directory' argument not a directory: '#{src}'"
+      end
     end
 
     #------------------------------
@@ -154,7 +180,7 @@ module Cmdtest
     #------------------------------
 
     def remove_file_tree(filename)
-      FileUtils.rm_rf(_cwd_path(filename))
+      Util.rm_rf(_cwd_path(filename))
     end
 
     #------------------------------
@@ -172,13 +198,13 @@ module Cmdtest
     #------------------------------
 
     def setenv(name, value)
-      @_env[name] = value
+      @_env_setenv[name] = [:setenv, value]
     end
 
     #------------------------------
 
     def unsetenv(name)
-      @_env.delete(name)
+      @_env_setenv[name] = [:unsetenv]
     end
 
     #------------------------------
@@ -361,9 +387,15 @@ module Cmdtest
       expected = files.flatten.sort
       #p [:xxx_files, xxx, actual, expected]
       _assert0 actual == expected do
-        _format_output(xxx.to_s.gsub(/_/, " ").gsub(/modified/, "changed"),
+        if @_runner.opts.diff
-                       actual.inspect + "\n",
+          _format_output(xxx.to_s.gsub(/_/, " ").gsub(/modified/, "changed"),
-                       expected.inspect + "\n")
+                         actual,
+                         expected)
+        else
+          _format_output(xxx.to_s.gsub(/_/, " ").gsub(/modified/, "changed"),
+                         actual.inspect + "\n",
+                         expected.inspect + "\n")
+        end
       end
     end
 
@@ -639,17 +671,26 @@ module Cmdtest
       end
     end
 
-    def _indented_lines(prefix, output)
+    def _to_lines(str)
-      case output
+      if Array === str
-      when Array
+        lines = str
-        lines = output
+      else
-      when String
+        lines = str.split(/\n/, -1)
-        lines = output.split(/\n/, -1)
         if lines[-1] == ""
           lines.pop
         elsif ! lines.empty?
           lines[-1] << "[[missing newline]]"
         end
+      end
+      return lines
+    end
+
+    def _indented_lines(prefix, output)
+      case output
+      when Array
+        lines = output
+      when String
+        lines = _to_lines(output)
       when Regexp
         lines = [output]
       else
@@ -672,8 +713,15 @@ module Cmdtest
     def _format_output(error, actual, expected)
       res = ""
       res << "ERROR: #{error}\n"
-      res << _indented_lines("       actual: ", actual)
+      if @_runner.opts.diff && (Array === expected || String === expected)
-      res << _indented_lines("       expect: ", expected)
+        expected_lines = _to_lines(expected)
+        actual_lines = _to_lines(actual)
+        diff = DiffLCS.new(expected_lines, actual_lines)
+        res << _indented_lines("  ", diff.to_a)
+      else
+        res << _indented_lines("       actual: ", actual)
+        res << _indented_lines("       expect: ", expected)
+      end
       return res
     end
 

lib/cmdtest/util.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # util.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -19,6 +19,8 @@
 # along with "cmdtest".  If not, see <http://www.gnu.org/licenses/>.
 #----------------------------------------------------------------------
 
+require "fileutils"
+
 module Cmdtest
   class Util
 
@@ -62,5 +64,11 @@ module Cmdtest
       RUBY_PLATFORM =~ /mswin32|mingw32/
     end
 
+    def self.rm_rf(path)
+      if File.exist?(path)
+        FileUtils.rm_r(path) # exception if failing
+      end
+    end
+
   end
 end

lib/cmdtest/workdir.rb

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------
 # workdir.rb
 #----------------------------------------------------------------------
-# Copyright 2002-2014 Johan Holmberg.
+# Copyright 2002-2016 Johan Holmberg.
 #----------------------------------------------------------------------
 # This file is part of "cmdtest".
 #
@@ -35,8 +35,8 @@ module Cmdtest
       @runner = runner
       @path = @runner.tmp_work_dir
       @hardlinkdir = File.join(@runner.tmp_dir, "hardlinks")
-      FileUtils.rm_rf(@path)
+      Util.rm_rf(@path)
-      FileUtils.rm_rf(@hardlinkdir)
+      Util.rm_rf(@hardlinkdir)
       FileUtils.mkdir_p(@path)
       @ignored_files = []
     end
@@ -93,8 +93,25 @@ module Cmdtest
     end
 
     def _ENV_strs(env)
-      # TODO: windows
+      env.keys.sort.map do |k|
-      env.keys.sort.map {|k| "export %s='%s'" % [k, env[k]] }
+        what = env[k][0]
+        case what
+        when :setenv
+          if Util.windows?
+            "set %s=%s" % [k, env[k][1]]
+          else
+            "export %s='%s'" % [k, env[k][1]]
+          end
+        when :unsetenv
+          if Util.windows?
+            "set %s=" % [k]
+          else
+            "unset %s" % [k]
+          end
+        else
+          raise "internal error"
+        end
+      end
     end
 
     def _chdir_str(dir)
@@ -139,7 +156,7 @@ module Cmdtest
       end
 
       File.open(_tmp_redirect_sh, "w") do |f|
-        f.puts _ENV_strs(@testcase._env)
+        f.puts _ENV_strs(@testcase._env_setenv)
         f.puts
         f.puts _chdir_str(@testcase._cwd)
         f.puts

python/Rakefile

@@ -0,0 +1,8 @@
+# -*- ruby -*-
+
+ENV["LC_ALL"] = "C" if RUBY_PLATFORM =~ /darwin/
+
+desc "run unit tests"
+task "test" do
+  sh "./cmdtest.py t"
+end

python/cmdtest.py

@@ -74,6 +74,13 @@ def subranges(n, arr):
     for i in range(0,len(arr)-n+1):
         yield arr[i:i+n]
 
+def flatten(seq):
+    for item in seq:
+        if isinstance(item, (list,tuple)):
+            yield from flatten(item)
+        else:
+            yield item
+
 def to_list(arg):
     return arg if isinstance(arg, list) else [arg]
 
@@ -104,6 +111,15 @@ def temp_chdir(path):
     finally:
         os.chdir(starting_directory)
 
+@contextlib.contextmanager
+def extra_sys_path(dname):
+    try:
+        old_sys_path = sys.path
+        sys.path.append(dname)
+        yield
+    finally:
+        sys.path = old_sys_path
+
 def progress(*args):
     print("###", "-" * 50, *args)
 
@@ -368,6 +384,7 @@ class TestCase:
     def __init__(self, tmpdir, statistics):
         self.__tmpdir = tmpdir
         self.__statistics = statistics
+        self.__always_ignored_files = set()
 
     def setup(self):
         pass
@@ -383,6 +400,9 @@ class TestCase:
         os.environ['PATH'] = os.pathsep.join((os.path.join(self.__tmpdir.top, dirpath),
                                               os.environ['PATH']))
 
+    def always_ignore_file(self, fname):
+        self.__always_ignored_files.add(fname)
+
     def import_file(self, src, tgt):
         mkdir_for(tgt)
         shutil.copy(os.path.join(ROOT_WD, src), tgt)
@@ -390,8 +410,8 @@ class TestCase:
     def create_file(self, fname, content, encoding='utf-8'):
         mkdir_for(fname)
         with open(fname, "w", encoding=encoding) as f:
-            if type(content) == list:
+            if isinstance(content, (list,tuple)):
-                for line in content:
+                for line in flatten(content):
                     print(line, file=f)
             else:
                 f.write(content)
@@ -409,7 +429,7 @@ class TestCase:
 
     def cmd(self, cmdline, timeout=None):
         tmpdir = self.__tmpdir
-        before = tmpdir.snapshot()
+        before = tmpdir.snapshot(self.__always_ignored_files)
         stdout_log = tmpdir.stdout_log()
         stderr_log = tmpdir.stderr_log()
         self._wait_for_new_second()
@@ -420,7 +440,7 @@ class TestCase:
                 err = subprocess.call(cmdline, stdout=stdout, stderr=stderr, shell=True, timeout=timeout)
             else:
                 err = 0
-        after = tmpdir.snapshot()
+        after = tmpdir.snapshot(self.__always_ignored_files)
 
         return Result(err, before, after,
                       File(stdout_log), File(stderr_log),
@@ -494,8 +514,9 @@ class FileInfo:
 #----------------------------------------------------------------------
 
 class FsSnapshot:
-    def __init__(self, topdir):
+    def __init__(self, topdir, ignored_files):
         self.topdir = topdir
+        self.ignored_files = ignored_files
         self.bypath = {}
         self._collect_files(DirInfo(topdir))
 
@@ -510,6 +531,8 @@ class FsSnapshot:
 
     def _collect_files(self, dirinfo):
         for entry in dirinfo.entries():
+            if entry.display_path in self.ignored_files:
+                continue
             self.bypath[entry.display_path] = entry
             if isinstance(entry, DirInfo):
                 self._collect_files(entry)
@@ -538,8 +561,8 @@ class Tmpdir:
     def timestamp_file(self):
         return os.path.join(self.logdir, "tmp.timestamp")
 
-    def snapshot(self):
+    def snapshot(self, ignored_files):
-        return FsSnapshot(self.top)
+        return FsSnapshot(self.top, ignored_files)
 
     def prepare_for_test(self, test_method_name):
         self.clear()
@@ -630,7 +653,9 @@ class Tfile:
 
         self.glob = dict()
         self.glob['TestCase'] = TestCase
-        exec(co, self.glob)
+        self.glob['__file__'] = os.path.abspath(filename)
+        with extra_sys_path(os.path.dirname(filename)):
+            exec(co, self.glob)
 
     def tclasses(self):
         for name in sorted(self.glob.keys()):
@@ -645,18 +670,20 @@ def cmdtest_in_dir(path):
         py_files = glob.glob("CMDTEST_*.py")
         return test_files(py_files)
 
-def test_files(py_files, selected_methods = set()):
+def test_files(py_files, selected_methods=None, quiet=False):
     statistics = Statistics()
     tmpdir = Tmpdir()
     for py_file in py_files:
         tfile = Tfile(py_file)
         for tclass in tfile.tclasses():
             statistics.classes += 1
-            progress(tclass.name())
+            if not quiet:
+                progress(tclass.name())
             for tmethod in tclass.tmethods():
                 if not selected_methods or tmethod.name() in selected_methods:
                     statistics.methods += 1
-                    progress(tmethod.name())
+                    if not quiet:
+                        progress(tmethod.name())
                     tmethod.run(tmpdir, statistics)
     return statistics
 
@@ -664,13 +691,17 @@ def parse_options():
     parser = argparse.ArgumentParser('cmdtest')
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="be more verbose")
+    parser.add_argument("-q", "--quiet", action="store_true",
+                        help="be more quiet")
     parser.add_argument("arg", nargs="*",
                         help="CMDTEST_*.py files / test methods")
     options = parser.parse_args()
     py_files = []
     selected_methods = set()
     for arg in options.arg:
-        if re.match(r'CMDTEST_.*\.py$', arg):
+        if os.path.isdir(arg):
+            py_files.extend(glob.glob('%s/CMDTEST_*.py' % arg))
+        elif re.match(r'CMDTEST_.*\.py$', os.path.basename(arg)):
             py_files.append(arg)
         else:
             selected_methods.add(arg)
@@ -684,10 +715,11 @@ def parse_options():
 
 def main():
     options, py_files, selected_methods = parse_options()
-    statistics = test_files(py_files, selected_methods)
+    statistics = test_files(py_files, selected_methods, options.quiet)
-    print()
+    if not options.quiet:
-    print(statistics)
+        print()
-    print()
+        print(statistics)
+        print()
     exit(0 if statistics.errors == 0 and statistics.fatals == 0 else 1)
 
 if __name__ == '__main__':

python/t/CMDTEST_as_module.py

@@ -0,0 +1,34 @@
+
+from os.path import dirname, abspath, join as path_join
+
+class TC_as_module(TestCase):
+
+    def setup(self):
+        self.always_ignore_file('subdir/tmp-cmdtest-python/')
+
+    def test_as_module(self):
+        self.create_file("subdir/CMDTEST_foo.py", [
+            'class TC_foo(TestCase):',
+            '    def test_01(self):',
+            '        with self.cmd("echo hello") as c:',
+            '            c.stdout_equal("hello\\n")',
+            '    def test_02(self):',
+            '        with self.cmd("echo world") as c:',
+            '            c.stdout_equal("world\\n")',
+            '    def test_03(self):',
+            '        with self.cmd("echo hello") as c:',
+            '            c.stdout_equal("world\\n")',
+        ])
+
+        dpath = dirname(abspath(__file__))
+        command = path_join(dpath, 'as_module.py')
+        with self.cmd(command + ' subdir') as c:
+            c.stdout_match([
+                r'--- ERROR: stdout_equal',
+                r'actual:',
+                r'     hello',
+                r'expect:',
+                r'     world',
+                r'Statistics\(classes=1, methods=3, command=3, errors=1, fatals=0\)',
+            ])
+            c.exit_nonzero()

python/t/CMDTEST_exit_nonzero.py

@@ -0,0 +1,42 @@
+
+from selftest_utils import SelftestUtils
+
+class TC_exit_nonzero(SelftestUtils, TestCase):
+
+    def test_exit_nonzero_CORRECT(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("false") as c:',
+            '    c.exit_nonzero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: false",
+            ])
+
+    def test_exit_nonzero_INCORRECT(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("true") as c:',
+            '    c.exit_nonzero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: true",
+                "--- ERROR: exit_nonzero",
+                "actual: 0",
+                "expect: <nonzero value>",
+                "",
+            ])
+            c.exit_nonzero()
+
+    def test_exit_nonzero_CORRECT_18(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("exit 18") as c:',
+            '    c.exit_nonzero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: exit 18",
+            ])

python/t/CMDTEST_exit_zero.py

@@ -0,0 +1,47 @@
+
+from selftest_utils import SelftestUtils
+
+class TC_exit_zero(SelftestUtils, TestCase):
+
+    def test_exit_zero_CORRECT(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("true") as c:',
+            '    c.exit_zero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: true",
+            ])
+
+    def test_exit_zero_INCORRECT(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("false") as c:',
+            '    c.exit_zero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: false",
+                "--- ERROR: exit_zero",
+                "actual: 1",
+                "expect: 0",
+                "",
+            ])
+            c.exit_nonzero()
+
+    def test_exit_zero_INCORRECT_18(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("exit 18") as c:',
+            '    c.exit_zero()',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: exit 18",
+                "--- ERROR: exit_zero",
+                "actual: 18",
+                "expect: 0",
+                "",
+            ])
+            c.exit_nonzero()

python/t/CMDTEST_stdout_equal.py

@@ -0,0 +1,71 @@
+
+from selftest_utils import SelftestUtils
+
+class TC_stdout_equal(SelftestUtils, TestCase):
+
+    def test_stdout_equal_CORRECT_EMPTY(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("true") as c:',
+            '    c.stdout_equal([',
+            '    ])',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: true",
+            ])
+
+    def test_stdout_equal_INCORRECT_EMPTY(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("echo hello") as c:',
+            '    c.stdout_equal([',
+            '    ])',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: echo hello",
+                "--- ERROR: stdout_equal",
+                "actual:",
+                "     hello",
+                "expect:",
+                "     <<empty>>",
+            ])
+            c.exit_nonzero()
+
+    def test_stdout_equal_CORRECT_2_LINES(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("echo hello && echo world") as c:',
+            '    c.stdout_equal([',
+            '        "hello",',
+            '        "world",',
+            '    ])',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: echo hello && echo world",
+            ])
+
+    def test_stdout_equal_INCORRECT_2_LINES(self):
+        self.create_CMDTEST_foo(
+            'with self.cmd("echo hello && echo world && echo MORE") as c:',
+            '    c.stdout_equal([',
+            '        "hello",',
+            '        "world",',
+            '    ])',
+        )
+
+        with self.cmd_cmdtest() as c:
+            c.stdout_equal([
+                "### cmdline: echo hello && echo world && echo MORE",
+                "--- ERROR: stdout_equal",
+                "actual:",
+                "     hello",
+                "     world",
+                "     MORE",
+                "expect:",
+                "     hello",
+                "     world",
+            ])
+            c.exit_nonzero()

python/t/as_module.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from os.path import dirname
+
+# import from 'cmdtest.py' in other directory
+sys.path.insert(0, dirname(dirname(os.path.abspath(__file__))))
+from cmdtest import cmdtest_in_dir, Statistics
+
+def main():
+    dirpath = sys.argv[1]
+    statistics = cmdtest_in_dir(dirpath)
+    print(statistics)
+    exit(0 if statistics.errors == 0 and statistics.fatals == 0 else 1)
+
+if __name__ == '__main__':
+    main()

python/t/selftest_utils.py

@@ -0,0 +1,30 @@
+
+import os
+from contextlib import contextmanager
+
+TOP = os.getcwd()
+
+class SelftestUtils:
+
+    def setup(self):
+        self.always_ignore_file('tmp-cmdtest-python/')
+
+    def create_CMDTEST_foo(self, *lines):
+        self.create_file("CMDTEST_foo.py", [
+            "class TC_foo(TestCase):",
+            "    def setup(self):",
+            "        #prepend_path #{BIN.inspect}",
+            "        #prepend_path #{PLATFORM_BIN.inspect}",
+            "        pass",
+            "",
+            "    def test_foo(self):",
+            [ "        " + line for line in lines],
+        ])
+
+    @contextmanager
+    def cmd_cmdtest(self, *args):
+        cmdtest = "%s/cmdtest.py" % TOP
+        command = "%s --quiet CMDTEST_foo.py" % cmdtest
+        cmdline = ' '.join([command] + list(args))
+        with self.cmd(cmdline) as c:
+            yield c

replace_strings.pl

@@ -0,0 +1,30 @@
+#!/usr/bin/env perl
+
+use strict;
+
+my @replace;
+
+while (@ARGV > 0 && $ARGV[0] =~ /^(\w+)=(.*)/) {
+    my ($old, $new) = ($1, $2);
+    push @replace, [$old, $new];
+    shift @ARGV;
+}
+
+if (@ARGV != 2) {
+    die "Usage: replace_strings K1=V1 ... Kn=Vn INFILE OUTFILE\n";
+}
+
+my ($infile, $outfile) = @ARGV;
+
+open(my $f, '<', $infile)  || die "open($infile)";
+open(my $g, '>', $outfile) || die "open($outfile)";
+while (my $line = <$f>) {
+    for my $entry (@replace) {
+        my ($old, $new) = @$entry;
+        $line =~ s/\$$old\$/$new/g;
+    }
+    print {$g} $line;
+}
+
+close($f) || die "close($infile)";
+close($g) || die "close($outfile)";

t/CMDTEST_import_directory.rb

@@ -0,0 +1,80 @@
+
+require "selftest_utils"
+
+class CMDTEST_import_directory < Cmdtest::Testcase
+
+  include SelftestUtils
+
+  #----------------------------------------
+  # import_directory
+  #----------------------------------------
+
+  def test_import_directory_ERROR
+    create_file "file1.dir/file1.txt", "This is file1.dir/file1.txt\n"
+    create_file "file2.dir/file2.txt", "This is file2.dir/file2.txt\n"
+
+    create_CMDTEST_foo [
+      "import_directory 'file1.dir', 'qwerty1.dir'",
+      "import_directory 'file2.dir', 'qwerty1.dir'",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal /CAUGHT EXCEPTION:/
+      stdout_equal /'import_directory' target argument already exist: 'qwerty1.dir'/
+      exit_nonzero
+    end
+  end
+
+  #----------------------------------------
+
+  def test_import_directory_DIFFERENT_DIRS
+    create_file "file1.dir/file1.txt", "This is file1.dir/file1.txt\n"
+    create_file "file2.dir/file2.txt", "This is file2.dir/file2.txt\n"
+
+    create_CMDTEST_foo [
+      "import_directory 'file1.dir', 'qwerty1.dir'",
+      "import_directory 'file2.dir', 'subdir/qwerty2.dir'",
+      "",
+      "cmd 'cat.rb qwerty1.dir/file1.txt subdir/qwerty2.dir/file2.txt' do",
+      "    stdout_equal [",
+      "        'This is file1.dir/file1.txt',",
+      "        'This is file2.dir/file2.txt',",
+      "    ]",
+      "end",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal [
+        "### cat.rb qwerty1.dir/file1.txt subdir/qwerty2.dir/file2.txt",
+      ]
+    end
+  end
+
+  #----------------------------------------
+
+  def test_import_directory_AFTER_CHDIR
+    create_file "file1.dir/file1.txt", "This is file1.dir/file1.txt\n"
+    create_file "file2.dir/file2.txt", "This is file2.dir/file2.txt\n"
+
+    create_CMDTEST_foo [
+      "dir_mkdir('dir')",
+      "chdir('dir')",
+      "import_directory 'file1.dir', 'qwerty1.dir'",
+      "import_directory 'file2.dir', 'subdir/qwerty2.dir'",
+      "",
+      "cmd 'cat.rb qwerty1.dir/file1.txt subdir/qwerty2.dir/file2.txt' do",
+      "    stdout_equal [",
+      "        'This is file1.dir/file1.txt',",
+      "        'This is file2.dir/file2.txt',",
+      "    ]",
+      "end",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal [
+        "### cat.rb qwerty1.dir/file1.txt subdir/qwerty2.dir/file2.txt",
+      ]
+    end
+  end
+
+end

t/CMDTEST_import_file.rb

@@ -9,6 +9,22 @@ class CMDTEST_import_file < Cmdtest::Testcase
   # import_file
   #----------------------------------------
 
+  def test_import_file_ERROR
+    create_file "file1.dir/empty.txt", ""
+
+    create_CMDTEST_foo [
+      "import_file 'file1.dir', 'qwerty1.dir'",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal /CAUGHT EXCEPTION:/
+      stdout_equal /'import_file' argument not a file: 'file1.dir'/
+      exit_nonzero
+    end
+  end
+
+  #----------------------------------------
+
   def test_import_file_DIFFERENT_DIRS
     create_file "file1.txt", "This is file1.txt\n"
     create_file "file2.txt", "This is file2.txt\n"

t/CMDTEST_options.rb

@@ -114,4 +114,16 @@ class CMDTEST_options < Cmdtest::Testcase
     end
   end
 
+  def test_option_help
+    cmd_cmdtest_verbose "-h" do
+      stdout_equal /^usage: cmdtest /
+      stdout_equal /^\s+-h, --help\s+show this help/
+    end
+
+    cmd_cmdtest_verbose "--help" do
+      stdout_equal /^usage: cmdtest /
+      stdout_equal /^\s+-h, --help\s+show this help/
+    end
+  end
+
 end

t/CMDTEST_raise.rb

@@ -0,0 +1,59 @@
+
+require "selftest_utils"
+
+class CMDTEST_raise < Cmdtest::Testcase
+
+  include SelftestUtils
+
+  def test_raise_TEST
+    create_file "CMDTEST_foo.rb", [
+      "class CMDTEST_foo < Cmdtest::Testcase",
+      "  def setup",
+      "    raise 'error in setup' if ENV['CMDTEST_RAISE'] == 'setup'",
+      "  end",
+      "",
+      "  def teardown",
+      "    raise 'error in teardown' if ENV['CMDTEST_RAISE'] == 'teardown'",
+      "  end",
+      "",
+      "  def test_foo",
+      "    raise 'error in test' if ENV['CMDTEST_RAISE'] == 'test'",
+      "    puts '123'",
+      "  end",
+      "end",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal [
+        "123",
+      ]
+    end
+
+    ENV['CMDTEST_RAISE'] = 'setup'
+    cmd_cmdtest do
+      exit_nonzero
+      stdout_equal /--- CAUGHT EXCEPTION:/
+      stdout_equal /---   error in setup/
+    end
+
+    ENV['CMDTEST_RAISE'] = 'test'
+    cmd_cmdtest do
+      exit_nonzero
+      stdout_equal /--- CAUGHT EXCEPTION:/
+      stdout_equal /---   error in test/
+    end
+
+    ENV['CMDTEST_RAISE'] = 'teardown'
+    cmd_cmdtest do
+      exit_nonzero
+      stdout_equal /--- CAUGHT EXCEPTION:/
+      stdout_equal /---   error in teardown/
+    end
+
+    ENV['CMDTEST_RAISE'] = nil
+    cmd_cmdtest do
+      stdout_equal "123\n"
+    end
+  end
+
+end

t/CMDTEST_readonly.rb

@@ -0,0 +1,33 @@
+
+require "selftest_utils"
+
+class CMDTEST_readonly < Cmdtest::Testcase
+
+  include SelftestUtils
+
+  def teardown
+    File.chmod(0755, 'tmp-cmdtest-2/top/work/a_subdir')
+  end
+
+  def test_readonly
+    create_CMDTEST_foo [
+      "cmd 'true.rb' do",
+      "end",
+      "Dir.mkdir('a_subdir')",
+      "File.open('a_subdir/file1', 'w') {|f| f.puts 123}",
+      "File.chmod(0, 'a_subdir')",
+    ]
+
+    cmd_cmdtest do
+      stdout_equal [
+        "### true.rb",
+      ]
+    end
+
+    cmd_cmdtest do
+      stderr_equal /Directory not empty/
+      exit_nonzero
+    end
+  end
+
+end

t/CMDTEST_setenv.rb

@@ -0,0 +1,83 @@
+require "selftest_utils"
+
+class CMDTEST_setenv < Cmdtest::Testcase
+
+  include SelftestUtils
+
+  #-----------------------------------
+
+  def test_setenv
+    #----------
+    create_CMDTEST_foo [
+      'cmd "env | grep TESTVAR1" do',
+      '  exit_nonzero', # no match in grep
+      'end',
+    ]
+    cmd_cmdtest do
+      comment "TESTVAR1 not set"
+      stdout_equal [
+        "### env | grep TESTVAR1",
+      ]
+    end
+
+    #----------
+    create_CMDTEST_foo [
+      'setenv "TESTVAR1", "123456"',
+      'cmd "env | grep TESTVAR1" do',
+      '  stdout_equal "TESTVAR1=123456\\n"',
+      'end',
+    ]
+    cmd_cmdtest do
+      comment "TESTVAR1 set by setenv"
+      stdout_equal [
+        "### env | grep TESTVAR1",
+      ]
+    end
+
+    cmd("env | grep TESTVAR1") do
+      comment "TESTVAR1 still unset on level1"
+      exit_nonzero # no match in grep
+    end
+  end
+
+  #-----------------------------------
+
+  def test_unsetenv
+    ENV['TESTVAR2'] = '987654'
+    #----------
+    create_CMDTEST_foo [
+      'cmd "env | grep TESTVAR2" do',
+      '  stdout_equal "TESTVAR2=987654\\n"',
+      'end',
+    ]
+    cmd_cmdtest do
+      comment "TESTVAR2 set from start"
+      stdout_equal [
+        "### env | grep TESTVAR2",
+      ]
+    end
+
+    #----------
+    create_CMDTEST_foo [
+      'unsetenv "TESTVAR2"',
+      'cmd "env | grep TESTVAR2" do',
+      '  exit_nonzero', # no match in grep
+      'end',
+    ]
+    cmd_cmdtest do
+      comment "TESTVAR2 unset by unsetenv"
+      stdout_equal [
+        "### env | grep TESTVAR2",
+      ]
+    end
+
+    cmd("env | grep TESTVAR2") do
+      comment "TESTVAR2 still set on level1"
+      stdout_equal [
+        "TESTVAR2=987654",
+      ]
+    end
+  end
+
+end
+

t/CMDTEST_stdxxx_equal.rb

@@ -181,6 +181,36 @@ class CMDTEST_stdxxx_equal < Cmdtest::Testcase
       end
     end
 
+    #----------------------------------------
+
+    ## methods: test_stdout_equal_DIFF test_stderr_equal_DIFF
+
+    define_method("test_#{stdxxx}_equal_DIFF") do
+
+      create_CMDTEST_foo [
+        "cmd 'echo_#{stdxxx}.rb --lines 11 33 44 55-changed 66 77 88' do",
+        "    #{stdxxx}_equal [ '11', '22', '33', '44', '55', '66', '77' ]",
+        "end",
+      ]
+
+      cmd_cmdtest_diff do
+        stdout_equal [
+          "### echo_#{stdxxx}.rb --lines 11 33 44 55-changed 66 77 88",
+          "--- ERROR: wrong #{stdxxx}",
+          "---     11",
+          "---   - 22",
+          "---     33",
+          "---     44",
+          "---   - 55",
+          "---   + 55-changed",
+          "---     66",
+          "---     77",
+          "---   + 88",
+        ]
+        exit_nonzero
+      end
+    end
+
     #----------------------------------------
     # stdxxx_not_equal
     #----------------------------------------

t/CMDTEST_summary.rb

@@ -62,6 +62,7 @@ class CMDTEST_summery < Cmdtest::Testcase
     ]
 
     cmd_cmdtest_verbose do
+      stdout_equal /^--- .* test classes,.*test methods,.*commands,.*errors,.*fatals\.$/
       stdout_equal /. 1 test classes/
       stdout_equal /. 5 test methods/
       stdout_equal /. 8 commands/
@@ -126,6 +127,7 @@ class CMDTEST_summery < Cmdtest::Testcase
     ]
 
     cmd_cmdtest_verbose do
+      stdout_equal /^### .* test classes,.*test methods,.*commands,.*errors,.*fatals\.$/
       stdout_equal /. 1 test classes/
       stdout_equal /. 5 test methods/
       stdout_equal /. 8 commands/

t/CMDTEST_teardown.rb

@@ -0,0 +1,42 @@
+
+require "selftest_utils"
+
+class CMDTEST_teardown < Cmdtest::Testcase
+  include SelftestUtils
+
+  def test_teardown
+    create_file "CMDTEST_foo.rb", [
+      "class CMDTEST_foo < Cmdtest::Testcase",
+      "  def setup",
+      "    puts 'setup: ' + Dir.pwd",
+      "  end",
+      "",
+      "  def teardown",
+      "    puts 'teardown: ' + Dir.pwd",
+      "  end",
+      "",
+      "  def test_foo",
+      "    puts 'test: ' + Dir.pwd",
+      "    Dir.mkdir('subdir')",
+      "    Dir.chdir('subdir') do",
+      "      puts 'test_subdir: ' + Dir.pwd",
+      "    end",
+      "  end",
+      "end",
+    ]
+
+    cwd = Dir.pwd
+    cmdtest = "#{TOP}/bin/cmdtest.rb"
+    command = "ruby %s --quiet" % _quote(cmdtest)
+    cmd(command) do
+      stdout_equal [
+        "setup: #{cwd}/tmp-cmdtest-2/top/work",
+        "test: #{cwd}/tmp-cmdtest-2/top/work",
+        "test_subdir: #{cwd}/tmp-cmdtest-2/top/work/subdir",
+        "teardown: #{cwd}/tmp-cmdtest-2/top/work",
+      ]
+    end
+
+  end
+
+end

t/bin/echo_stderr.rb

@@ -1,3 +1,9 @@
 #!/usr/bin/ruby
 
-STDERR.puts ARGV.join(" ")
+if ARGV.size > 0 && ARGV[0] == "--lines"
+    for arg in ARGV[1..-1]
+        STDERR.puts arg
+    end
+else
+    STDERR.puts ARGV.join(" ")
+end

t/bin/echo_stdout.rb

@@ -1,3 +1,9 @@
 #!/usr/bin/ruby
 
-puts ARGV.join(" ")
+if ARGV.size > 0 && ARGV[0] == "--lines"
+    for arg in ARGV[1..-1]
+        puts arg
+    end
+else
+    puts ARGV.join(" ")
+end

t/selftest_utils.rb

@@ -50,6 +50,15 @@ module SelftestUtils
     end
   end
 
+  def cmd_cmdtest_diff(*args)
+    cmdtest = "#{TOP}/bin/cmdtest.rb"
+    command = "ruby %s --quiet --diff" % _quote(cmdtest)
+    cmd(command, *args) do
+      comment "running local cmdtest --diff"
+      yield
+    end
+  end
+
   def cmd_cmdtest_verbose(*args)
     cmdtest = "#{TOP}/bin/cmdtest.rb"
     command = "ruby %s %s" % [