921 lines
41 KiB
HTML
921 lines
41 KiB
HTML
<?xml version="1.0" encoding="utf-8" ?>
|
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="Docutils 0.14: http://docutils.sourceforge.net/" />
|
|
<title>Cmdtest User Guide</title>
|
|
<style type="text/css">
|
|
|
|
/*
|
|
:Author: David Goodger (goodger@python.org)
|
|
:Id: $Id: html4css1.css 5631 2008-08-24 13:01:23Z goodger $
|
|
:Copyright: This stylesheet has been placed in the public domain.
|
|
|
|
Default cascading style sheet for the HTML output of Docutils.
|
|
|
|
See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to
|
|
customize this style sheet.
|
|
*/
|
|
|
|
/* used to remove borders from tables and images */
|
|
.borderless, table.borderless td, table.borderless th {
|
|
border: 0 }
|
|
|
|
table.borderless td, table.borderless th {
|
|
/* Override padding for "table.docutils td" with "! important".
|
|
The right padding separates the table cells. */
|
|
padding: 0 0.5em 0 0 ! important }
|
|
|
|
.first {
|
|
/* Override more specific margin styles with "! important". */
|
|
margin-top: 0 ! important }
|
|
|
|
.last, .with-subtitle {
|
|
margin-bottom: 0 ! important }
|
|
|
|
.hidden {
|
|
display: none }
|
|
|
|
a.toc-backref {
|
|
text-decoration: none ;
|
|
color: black }
|
|
|
|
blockquote.epigraph {
|
|
margin: 2em 5em ; }
|
|
|
|
dl.docutils dd {
|
|
margin-bottom: 0.5em }
|
|
|
|
/* Uncomment (and remove this text!) to get bold-faced definition list terms
|
|
dl.docutils dt {
|
|
font-weight: bold }
|
|
*/
|
|
|
|
div.abstract {
|
|
margin: 2em 5em }
|
|
|
|
div.abstract p.topic-title {
|
|
font-weight: bold ;
|
|
text-align: center }
|
|
|
|
div.admonition, div.attention, div.caution, div.danger, div.error,
|
|
div.hint, div.important, div.note, div.tip, div.warning {
|
|
margin: 2em ;
|
|
border: medium outset ;
|
|
padding: 1em }
|
|
|
|
div.admonition p.admonition-title, div.hint p.admonition-title,
|
|
div.important p.admonition-title, div.note p.admonition-title,
|
|
div.tip p.admonition-title {
|
|
font-weight: bold ;
|
|
font-family: sans-serif }
|
|
|
|
div.attention p.admonition-title, div.caution p.admonition-title,
|
|
div.danger p.admonition-title, div.error p.admonition-title,
|
|
div.warning p.admonition-title {
|
|
color: red ;
|
|
font-weight: bold ;
|
|
font-family: sans-serif }
|
|
|
|
/* Uncomment (and remove this text!) to get reduced vertical space in
|
|
compound paragraphs.
|
|
div.compound .compound-first, div.compound .compound-middle {
|
|
margin-bottom: 0.5em }
|
|
|
|
div.compound .compound-last, div.compound .compound-middle {
|
|
margin-top: 0.5em }
|
|
*/
|
|
|
|
div.dedication {
|
|
margin: 2em 5em ;
|
|
text-align: center ;
|
|
font-style: italic }
|
|
|
|
div.dedication p.topic-title {
|
|
font-weight: bold ;
|
|
font-style: normal }
|
|
|
|
div.figure {
|
|
margin-left: 2em ;
|
|
margin-right: 2em }
|
|
|
|
div.footer, div.header {
|
|
clear: both;
|
|
font-size: smaller }
|
|
|
|
div.line-block {
|
|
display: block ;
|
|
margin-top: 1em ;
|
|
margin-bottom: 1em }
|
|
|
|
div.line-block div.line-block {
|
|
margin-top: 0 ;
|
|
margin-bottom: 0 ;
|
|
margin-left: 1.5em }
|
|
|
|
div.sidebar {
|
|
margin: 0 0 0.5em 1em ;
|
|
border: medium outset ;
|
|
padding: 1em ;
|
|
background-color: #ffffee ;
|
|
width: 40% ;
|
|
float: right ;
|
|
clear: right }
|
|
|
|
div.sidebar p.rubric {
|
|
font-family: sans-serif ;
|
|
font-size: medium }
|
|
|
|
div.system-messages {
|
|
margin: 5em }
|
|
|
|
div.system-messages h1 {
|
|
color: red }
|
|
|
|
div.system-message {
|
|
border: medium outset ;
|
|
padding: 1em }
|
|
|
|
div.system-message p.system-message-title {
|
|
color: red ;
|
|
font-weight: bold }
|
|
|
|
div.topic {
|
|
margin: 2em }
|
|
|
|
h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
|
|
h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
|
|
margin-top: 0.4em }
|
|
|
|
h1.title {
|
|
text-align: center }
|
|
|
|
h2.subtitle {
|
|
text-align: center }
|
|
|
|
hr.docutils {
|
|
width: 75% }
|
|
|
|
img.align-left {
|
|
clear: left }
|
|
|
|
img.align-right {
|
|
clear: right }
|
|
|
|
ol.simple, ul.simple {
|
|
margin-bottom: 1em }
|
|
|
|
ol.arabic {
|
|
list-style: decimal }
|
|
|
|
ol.loweralpha {
|
|
list-style: lower-alpha }
|
|
|
|
ol.upperalpha {
|
|
list-style: upper-alpha }
|
|
|
|
ol.lowerroman {
|
|
list-style: lower-roman }
|
|
|
|
ol.upperroman {
|
|
list-style: upper-roman }
|
|
|
|
p.attribution {
|
|
text-align: right ;
|
|
margin-left: 50% }
|
|
|
|
p.caption {
|
|
font-style: italic }
|
|
|
|
p.credits {
|
|
font-style: italic ;
|
|
font-size: smaller }
|
|
|
|
p.label {
|
|
white-space: nowrap }
|
|
|
|
p.rubric {
|
|
font-weight: bold ;
|
|
font-size: larger ;
|
|
color: maroon ;
|
|
text-align: center }
|
|
|
|
p.sidebar-title {
|
|
font-family: sans-serif ;
|
|
font-weight: bold ;
|
|
font-size: larger }
|
|
|
|
p.sidebar-subtitle {
|
|
font-family: sans-serif ;
|
|
font-weight: bold }
|
|
|
|
p.topic-title {
|
|
font-weight: bold }
|
|
|
|
pre.address {
|
|
margin-bottom: 0 ;
|
|
margin-top: 0 ;
|
|
font: inherit }
|
|
|
|
tt.literal {
|
|
color: #b33;
|
|
}
|
|
|
|
pre.literal-block, pre.doctest-block {
|
|
margin-left: 2em ;
|
|
x-background-color: #ddd;
|
|
background-color: #fec;
|
|
padding: 0.5em;
|
|
outline-style: dashed;
|
|
outline-width: 0.1em;
|
|
margin-right: 2em }
|
|
|
|
span.classifier {
|
|
font-family: sans-serif ;
|
|
font-style: oblique }
|
|
|
|
span.classifier-delimiter {
|
|
font-family: sans-serif ;
|
|
font-weight: bold }
|
|
|
|
span.interpreted {
|
|
font-family: sans-serif }
|
|
|
|
span.option {
|
|
white-space: nowrap }
|
|
|
|
span.pre {
|
|
white-space: pre }
|
|
|
|
span.problematic {
|
|
color: red }
|
|
|
|
span.section-subtitle {
|
|
/* font-size relative to parent (h1..h6 element) */
|
|
font-size: 80% }
|
|
|
|
table.citation {
|
|
border-left: solid 1px gray;
|
|
margin-left: 1px }
|
|
|
|
table.docinfo {
|
|
margin: 2em 4em }
|
|
|
|
table.docutils {
|
|
margin-top: 0.5em ;
|
|
margin-bottom: 0.5em }
|
|
|
|
table.footnote {
|
|
border-left: solid 1px black;
|
|
margin-left: 1px }
|
|
|
|
table.docutils td, table.docutils th,
|
|
table.docinfo td, table.docinfo th {
|
|
padding-left: 0.5em ;
|
|
padding-right: 0.5em ;
|
|
vertical-align: top }
|
|
|
|
table.docutils th.field-name, table.docinfo th.docinfo-name {
|
|
font-weight: bold ;
|
|
text-align: left ;
|
|
white-space: nowrap ;
|
|
padding-left: 0 }
|
|
|
|
h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
|
|
h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
|
|
font-size: 100% }
|
|
|
|
ul.auto-toc {
|
|
list-style-type: none }
|
|
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="document" id="cmdtest-user-guide">
|
|
<h1 class="title">Cmdtest User Guide</h1>
|
|
|
|
<div class="contents topic" id="contents">
|
|
<p class="topic-title first">Contents</p>
|
|
<ul class="simple">
|
|
<li><a class="reference internal" href="#introduction" id="id1">Introduction</a></li>
|
|
<li><a class="reference internal" href="#a-simple-example" id="id2">A simple example</a></li>
|
|
<li><a class="reference internal" href="#reporting-format" id="id3">Reporting format</a></li>
|
|
<li><a class="reference internal" href="#structure-of-a-test-file" id="id4">Structure of a test-file</a></li>
|
|
<li><a class="reference internal" href="#structure-of-a-test-method" id="id5">Structure of a test-method</a></li>
|
|
<li><a class="reference internal" href="#work-directory" id="id6">Work directory</a></li>
|
|
<li><a class="reference internal" href="#specifying-files-directories" id="id7">Specifying files / directories</a></li>
|
|
<li><a class="reference internal" href="#path-handling" id="id8">PATH handling</a></li>
|
|
<li><a class="reference internal" href="#matching-standard-output-content" id="id9">Matching standard output content</a></li>
|
|
<li><a class="reference internal" href="#invoking-cmdtest" id="id10">Invoking <tt class="docutils literal">cmdtest</tt></a><ul>
|
|
<li><a class="reference internal" href="#options" id="id11">Options</a></li>
|
|
<li><a class="reference internal" href="#commandline-examples" id="id12">Commandline Examples</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#reference-part" id="id13">Reference Part</a><ul>
|
|
<li><a class="reference internal" href="#cmd" id="id14">cmd</a></li>
|
|
<li><a class="reference internal" href="#assertions-exit-status" id="id15">Assertions - exit status</a></li>
|
|
<li><a class="reference internal" href="#assertions-files" id="id16">Assertions - files</a></li>
|
|
<li><a class="reference internal" href="#assertions-stdout-stderr-file-content" id="id17">Assertions - stdout/stderr/file content</a></li>
|
|
<li><a class="reference internal" href="#assertions-misc" id="id18">Assertions - misc</a></li>
|
|
<li><a class="reference internal" href="#helper-functions" id="id19">Helper functions</a></li>
|
|
<li><a class="reference internal" href="#deprecated-helper-functions" id="id20">Deprecated helper functions</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="section" id="introduction">
|
|
<h1><a class="toc-backref" href="#id1">Introduction</a></h1>
|
|
<p>Cmdtest is a <a class="reference external" href="http://en.wikipedia.org/wiki/Unit_testing">unit testing</a> framework for testing commands (executable programs).
|
|
In other test frameworks the "unit" tested is often a class (e.g. in Java's <a class="reference external" href="http://en.wikipedia.org/wiki/JUnit">JUnit</a> or
|
|
Ruby's <a class="reference external" href="https://github.com/test-unit/test-unit">Test::Unit</a>), but in Cmdtest the unit is an executable. Apart from this
|
|
difference Cmdtest borrows many ideas from the other frameworks.
|
|
The program <tt class="docutils literal">cmdtest</tt> runs the tests and reports the success or failure
|
|
in different ways, e.g. by writing to standard output or producing an XML-file on
|
|
Ant/JUnit format. The testcases are written in Ruby code. Assertions can
|
|
be made about the side effects performed by a command:</p>
|
|
<ul class="simple">
|
|
<li>the exit status</li>
|
|
<li>the content of standard output</li>
|
|
<li>the content of standard error</li>
|
|
<li>newly created/removed/changed files, or other changes to the
|
|
filesystem</li>
|
|
<li>encoding of a file (eg. UTF-8)</li>
|
|
</ul>
|
|
</div>
|
|
<div class="section" id="a-simple-example">
|
|
<h1><a class="toc-backref" href="#id2">A simple example</a></h1>
|
|
<pre class="literal-block">
|
|
$ cat CMDTEST_example.rb
|
|
class CMDTEST_example < Cmdtest::Testcase
|
|
|
|
def test_hello_world
|
|
cmd "echo hello" do
|
|
stdout_equal "hello\n"
|
|
end
|
|
|
|
cmd "echo world" do
|
|
stdout_equal "world\n"
|
|
end
|
|
end
|
|
|
|
def test_touch_and_exit
|
|
cmd "touch foo.txt ; exit 7" do
|
|
created_files "foo.txt"
|
|
exit_status 7
|
|
end
|
|
end
|
|
|
|
end
|
|
</pre>
|
|
<p>This example shows the basic structure of a testcase file. First we make a
|
|
subclass of <tt class="docutils literal"><span class="pre">Cmdtest::Testcase</span></tt>. All methods of the new class with a
|
|
name like <tt class="docutils literal">test_*</tt> will be considered testcases.
|
|
Inside a method we can call the <tt class="docutils literal">cmd</tt> method. It will
|
|
execute the command given as argument and then check the assertions
|
|
given in the do-block. When <tt class="docutils literal">cmdtest</tt> is run, it will find all
|
|
<tt class="docutils literal"><span class="pre">CMDTEST_*.rb</span></tt> files in the current directory and run the tests in
|
|
the files (see the <a class="reference internal" href="#invoking-cmdtest">Invoking cmdtest</a> section for more details
|
|
on <tt class="docutils literal"><span class="pre">CMDTEST_*.rb</span></tt> file selection).
|
|
The output looks like:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest
|
|
### ======================================== CMDTEST_example.rb
|
|
### ---------------------------------------- CMDTEST_example
|
|
### ........................................ test_hello_world
|
|
### echo hello
|
|
### echo world
|
|
### ........................................ test_touch_and_exit
|
|
### touch foo.txt ; exit 7
|
|
|
|
### 1 test classes, 2 test methods, 3 commands, 0 errors, 0 fatals.
|
|
</pre>
|
|
<p>If we change "7" to "8", "foo" to "bar" and "world" to "WORLD" in
|
|
the example, we get the following errors:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest
|
|
### ======================================== CMDTEST_example.rb
|
|
### ---------------------------------------- CMDTEST_example
|
|
### ........................................ test_hello_world
|
|
### echo hello
|
|
### echo WORLD
|
|
--- ERROR: wrong stdout
|
|
--- actual: WORLD
|
|
--- expect: world
|
|
### ........................................ test_touch_and_exit
|
|
### touch bar.txt ; exit 8
|
|
--- ERROR: created files
|
|
--- actual: ["bar.txt"]
|
|
--- expect: ["foo.txt"]
|
|
--- ERROR: expected 7 exit status, got 8
|
|
|
|
--- 1 test classes, 2 test methods, 3 commands, 2 errors, 0 fatals.
|
|
</pre>
|
|
<p>The following sections will describe in more detail what can be done
|
|
with Cmdtest. See also the <a class="reference external" href="../examples">examples directory</a> of the Cmdtest project,
|
|
where some larger examples of Cmdtest usage can be found.</p>
|
|
</div>
|
|
<div class="section" id="reporting-format">
|
|
<h1><a class="toc-backref" href="#id3">Reporting format</a></h1>
|
|
<p>Normally Cmdtest writes lines on standard output to show the progress of the
|
|
testing. As long as no error occurs, the lines will be prefixed by
|
|
"###". Error messages will instead have a "---" prefix. This makes it easy
|
|
to spot errors just by looking in the left margin. Each call to <tt class="docutils literal">cmd</tt>
|
|
will give one line on standard output. Normally the command executed will be
|
|
shown (after the "###" prefix). But one can also replace the string
|
|
written by calling the <tt class="docutils literal">comment</tt> method inside the do-block of a <tt class="docutils literal">cmd</tt>
|
|
call.</p>
|
|
<p>When an error occurs in a test-method, the rest of the method will be
|
|
skipped. But all errors occurring at the same command will be reported.</p>
|
|
<p>Cmdtest can also be directed to write an XML file on the same format as
|
|
that used by Ant/JUnit. This makes it possible to use Cmdtest together
|
|
with <a class="reference external" href="http://en.wikipedia.org/wiki/Continuous_integration">continuous integration</a> servers like <a class="reference external" href="http://jenkins-ci.org">Jenkins</a>.</p>
|
|
<p>The exit status of <tt class="docutils literal">cmdtest</tt> will be non-zero if some errors occurred,
|
|
otherwise zero. If errors should not affect exit code, the
|
|
command line option <tt class="docutils literal"><span class="pre">--no-exit-code</span></tt> can be used.</p>
|
|
</div>
|
|
<div class="section" id="structure-of-a-test-file">
|
|
<h1><a class="toc-backref" href="#id4">Structure of a test-file</a></h1>
|
|
<p>Each test-file can contain one or more subclasses to
|
|
<tt class="docutils literal"><span class="pre">Cmdtest::Testcase</span></tt>. The methods that are special are:</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">test_*</tt></dt>
|
|
<dd>These are the methods that will run tests.
|
|
For each method, a newly created object of the class will be used.</dd>
|
|
<dt><tt class="docutils literal">setup</tt></dt>
|
|
<dd>This method is called before each <tt class="docutils literal">test_*</tt> method is called.
|
|
It gives the user a chance to initialize the "environment" of all
|
|
the <tt class="docutils literal">test_*</tt> methods of the class. It can be seen as a "user level"
|
|
constructor.</dd>
|
|
<dt><tt class="docutils literal">teardown</tt></dt>
|
|
<dd>This method is called after each <tt class="docutils literal">test_*</tt> method was called. It
|
|
gives the user a chance to cleanup the "environment" of all the
|
|
<tt class="docutils literal">test_*</tt> methods of the class, e.g. release some resource acquired
|
|
by the <tt class="docutils literal">setup</tt> method. It can be seen as a "user level" destructor.</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="structure-of-a-test-method">
|
|
<h1><a class="toc-backref" href="#id5">Structure of a test-method</a></h1>
|
|
<p>Each test-method (named <tt class="docutils literal">test_*</tt>) should contain a number of calls to
|
|
the <tt class="docutils literal">cmd</tt> method. Inside the do-block of the <tt class="docutils literal">cmd</tt> calls, a number of
|
|
assertions can be made about the outcome of the command. The simplest
|
|
possible call looks like:</p>
|
|
<pre class="literal-block">
|
|
cmd "true" do
|
|
end
|
|
</pre>
|
|
<p>Here no explicit assertions have been given. In that case Cmdtest
|
|
applies some implicit assertions. The code above is equivalent to the
|
|
following more explicit one:</p>
|
|
<pre class="literal-block">
|
|
cmd "true" do
|
|
exit_zero
|
|
stdout_equal ""
|
|
stderr_equal ""
|
|
created_files []
|
|
changed_files []
|
|
removed_files []
|
|
end
|
|
</pre>
|
|
<p>The idea is that all differences in behaviour from the trivial <tt class="docutils literal">true</tt>
|
|
command should be described as an assertion in the do-block. The list
|
|
of possible assertions includes: <tt class="docutils literal">exit_zero</tt>, <tt class="docutils literal">exit_nonzero</tt>,
|
|
<tt class="docutils literal">exit_status</tt>, <tt class="docutils literal">created_files</tt>, <tt class="docutils literal">changed_files</tt>, <tt class="docutils literal">removed_files</tt>,
|
|
<tt class="docutils literal">written_files</tt>, <tt class="docutils literal">affected_files</tt>, <tt class="docutils literal">file_equal</tt>, <tt class="docutils literal">file_encoding</tt>,
|
|
<tt class="docutils literal">stdout_equal</tt> and <tt class="docutils literal">stderr_equal</tt>.</p>
|
|
<p>In addition to the assertions there are other helper-functions to set
|
|
up the "environment" for the commands and assertions. An example is
|
|
the creation of files:</p>
|
|
<pre class="literal-block">
|
|
...
|
|
create_file "foo.txt", "abc\ndef\n"
|
|
cmd "cat -n foo.txt" do
|
|
stdout_equal [
|
|
" 1\tabc",
|
|
" 2\tdef",
|
|
]
|
|
end
|
|
...
|
|
</pre>
|
|
<p>The list of such helper functions includes:
|
|
<tt class="docutils literal">create_file</tt>, <tt class="docutils literal">touch_file</tt>, <tt class="docutils literal">import_file</tt> , <tt class="docutils literal">import_directory</tt> and <tt class="docutils literal">ignore_file</tt>.
|
|
Beside these methods the test can of course also contain arbitrary Ruby-code.</p>
|
|
<p>Some test are not applicable under all circumstances. A test can for example be Linux specific.
|
|
Then the function <tt class="docutils literal">skip_test</tt> can be used, like this:</p>
|
|
<pre class="literal-block">
|
|
def test_linux_stuff
|
|
if RUBY_PLATFORM !~ /linux/
|
|
skip_test "not on linux"
|
|
end
|
|
... the actual tests ...
|
|
end
|
|
</pre>
|
|
</div>
|
|
<div class="section" id="work-directory">
|
|
<h1><a class="toc-backref" href="#id6">Work directory</a></h1>
|
|
<p>All tests are performed in a "clean" temporary directory, here called the "work directory".
|
|
When the <tt class="docutils literal">setup</tt>, <tt class="docutils literal">test_*</tt> and <tt class="docutils literal">teardown</tt> methods are called the current directory
|
|
will be the "work directory" (unless <tt class="docutils literal">chdir</tt> is called by the methods themselves).</p>
|
|
<p>Several of the assertions and helper functions take filename arguments
|
|
that are evaluated relative to the "work directory" (or sometimes the
|
|
current directory if they differ).</p>
|
|
<p>Cmdtest implements parallel execution of test methods by running several
|
|
"slave processes", started by a tool like <a class="reference external" href="http://www.gnu.org/software/parallel/">GNU Parallel</a>.</p>
|
|
<p>Methods such as <tt class="docutils literal">File.open</tt> and <tt class="docutils literal">Dir.chdir</tt> that depend on the
|
|
"current directory" can be used in the test methods, since each slave
|
|
is a process of its own (an earlier version of Cmdtest used Ruby threads
|
|
and adviced against using such methods).</p>
|
|
</div>
|
|
<div class="section" id="specifying-files-directories">
|
|
<h1><a class="toc-backref" href="#id7">Specifying files / directories</a></h1>
|
|
<p>Several methods take files or directories as argument (e.g.
|
|
<tt class="docutils literal">created_files</tt>, <tt class="docutils literal">modified_files</tt> and <tt class="docutils literal">ignore_file</tt>). Instead of
|
|
having two sets of methods, one for files and one for directories, an
|
|
argument with a trailing "/" denotes a directory:</p>
|
|
<pre class="literal-block">
|
|
created_files "build/" # the directory "build"
|
|
created_files "build" # the file "build"
|
|
|
|
ignore_file "build/" # the directory "build" (and everything below)
|
|
ignore_file "build" # the file "build"
|
|
</pre>
|
|
<p>As can be seen in the example above, the <tt class="docutils literal">ignore_file</tt> method is
|
|
special, because an ignored directory means that all files below the directory are
|
|
ignored too. Another peculiarity with <tt class="docutils literal">ignore_file</tt> is that the
|
|
argument can be a Regexp:</p>
|
|
<pre class="literal-block">
|
|
ignore_file /\.o$/ # all files *.o
|
|
</pre>
|
|
<p>This is quite natural, since the "job" of <tt class="docutils literal">ignore_file</tt> is to single
|
|
out a subset of all files.</p>
|
|
</div>
|
|
<div class="section" id="path-handling">
|
|
<h1><a class="toc-backref" href="#id8">PATH handling</a></h1>
|
|
<p>Cmdtest is used to test commands, so an important question is how the
|
|
commands are found and executed. Normally commands are found via the
|
|
<tt class="docutils literal">PATH</tt> environment variable, and Cmdtest is no exception. The commands
|
|
executed in the <tt class="docutils literal">cmd</tt> calls are evaluated in a shell script (on
|
|
UN*X) or in a BAT file (on Windows). The <tt class="docutils literal">PATH</tt> in effect when
|
|
<tt class="docutils literal">cmdtest</tt> is invoked is kept intact, with one addition: the current
|
|
directory at the time of invocation is prepended to the <tt class="docutils literal">PATH</tt>. If
|
|
further changes to the <tt class="docutils literal">PATH</tt> are needed the methods <tt class="docutils literal">prepend_path</tt>,
|
|
<tt class="docutils literal">prepend_local_path</tt> or <tt class="docutils literal">set_path</tt> can be used. Such path modifications
|
|
does not survive between test methods. Each new test method starts with the
|
|
original value of <tt class="docutils literal">PATH</tt>.</p>
|
|
</div>
|
|
<div class="section" id="matching-standard-output-content">
|
|
<h1><a class="toc-backref" href="#id9">Matching standard output content</a></h1>
|
|
<p>An assertion like <tt class="docutils literal">stdout_equal</tt> compares the actual standard output of a
|
|
command with the expected outcome. The expected value can be specified
|
|
in different ways, and is best explained by example:</p>
|
|
<pre class="literal-block">
|
|
cmd "echo hello ; echo world" do
|
|
stdout_equal "hello\nworld\n" # 1
|
|
stdout_equal [ # 2
|
|
"hello",
|
|
"world"
|
|
]
|
|
stdout_equal /orld/ # 3
|
|
stdout_equal [ # 4
|
|
"hello",
|
|
/world|earth/
|
|
]
|
|
end
|
|
</pre>
|
|
<p>In the example we see how the content can be specified:</p>
|
|
<ol class="arabic simple">
|
|
<li>as a string, with a newline (<tt class="docutils literal">\n</tt>) character for each new line</li>
|
|
<li>as an array of lines</li>
|
|
<li>as a regexp that should match the file content given as a string</li>
|
|
<li>as an array of lines where some lines should match a regexp rather than be compared
|
|
for string equality</li>
|
|
</ol>
|
|
</div>
|
|
<div class="section" id="invoking-cmdtest">
|
|
<h1><a class="toc-backref" href="#id10">Invoking <tt class="docutils literal">cmdtest</tt></a></h1>
|
|
<p><tt class="docutils literal">cmdtest</tt> can be called without any arguments at all. It will then look
|
|
for <tt class="docutils literal"><span class="pre">CMDTEST_*.rb</span></tt> files in the following places:</p>
|
|
<ol class="arabic simple">
|
|
<li>first <tt class="docutils literal"><span class="pre">t/CMDTEST_*.rb</span></tt></li>
|
|
<li>second <tt class="docutils literal"><span class="pre">test/CMDTEST_*.rb</span></tt></li>
|
|
<li>otherwise <tt class="docutils literal"><span class="pre">CMDTEST_*.rb</span></tt></li>
|
|
</ol>
|
|
<p>If some command line arguments have been given, <tt class="docutils literal">cmdtest</tt> will use
|
|
them instead of searching by itself. Some examples:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest CMDTEST_foo.rb # just one file
|
|
$ cmdtest CMDTEST_foo.rb CMDTEST_bar.rb # two files
|
|
$ cmdtest t # all CMDTEST_*.rb files in "t" dir
|
|
$ cmdtest . t # all CMDTEST_*.rb files in both dirs
|
|
</pre>
|
|
<p>In addition to test files, the command line can also contain options
|
|
and testcase selectors. The general format is:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest [options] [files] [selectors]
|
|
</pre>
|
|
<p>The options are describe in a separate section below. The selectors are
|
|
regular expressions that are used to match the names of the test
|
|
methods. It is best illustrated by an example:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest examples "/stdin|stdout/"
|
|
</pre>
|
|
<p>This command will find all files matching <tt class="docutils literal"><span class="pre">examples/CMDTEST_*.rb</span></tt>,
|
|
and run all test methods whose names either contain the string "stdin"
|
|
or "stdout". As can be seen in the example, the regular expression may
|
|
need protection from expansion by the shell (that is the reason for
|
|
the quotes in the example). But the example can also be written:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest examples /stdin/ /stdout/
|
|
</pre>
|
|
<p>For more examples of command line usage, see the section <a class="reference internal" href="#commandline-examples">Commandline Examples</a> below.</p>
|
|
<div class="section" id="options">
|
|
<h2><a class="toc-backref" href="#id11">Options</a></h2>
|
|
<p>The available options can be seen by using the <tt class="docutils literal"><span class="pre">-h</span></tt> option:</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest -h
|
|
usage: cmdtest [-h] [--shortversion] [--version] [-q] [-v] [--diff] [--no-diff]
|
|
[--fast] [-j N] [--test TEST] [--xml FILE] [--no-exit-code]
|
|
[--stop-on-error] [-i] [--slave SLAVE]
|
|
[arg [arg ...]]
|
|
|
|
positional arguments:
|
|
arg testfile or pattern
|
|
|
|
optional arguments:
|
|
-h, --help show this help message and exit
|
|
--shortversion show just version number
|
|
--version show version
|
|
-q, --quiet be more quiet
|
|
-v, --verbose be more verbose
|
|
--diff diff output (default)
|
|
--no-diff old non-diff output
|
|
--fast run fast without waiting for unique mtime:s
|
|
-j N, --parallel N build in parallel
|
|
--test TEST only run named test
|
|
--xml FILE write summary on JUnit format
|
|
--no-exit-code exit with 0 status even after errors
|
|
--stop-on-error exit after first error
|
|
-i, --incremental incremental mode
|
|
--slave SLAVE run in slave mode
|
|
</pre>
|
|
</div>
|
|
<div class="section" id="commandline-examples">
|
|
<h2><a class="toc-backref" href="#id12">Commandline Examples</a></h2>
|
|
<p>This section is a collection of examples of how Cmdtest can be used.</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest
|
|
</pre>
|
|
<p>This is the most basic usage. All testcase files found (by the
|
|
algorithm described earlier) will be executed.</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest -i
|
|
</pre>
|
|
<p>Only run the test methods that have failed earlier, or have changed.
|
|
This is not a full-blown "make system", but may still be useful when
|
|
developing the tests.</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest /stdout/
|
|
</pre>
|
|
<p>Run all test methods matching the regular expression given.</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest examples
|
|
</pre>
|
|
<p>Run all tests found in test files in the "examples" directory
|
|
(i.e. <tt class="docutils literal"><span class="pre">examples/CMDTEST_*.rb</span></tt>).</p>
|
|
<pre class="literal-block">
|
|
$ cmdtest --xml=reports/test-foo.xml
|
|
</pre>
|
|
<p>Write an XML-summary to the specified file. The file uses the same
|
|
format as <a class="reference external" href="http://en.wikipedia.org/wiki/JUnit">JUnit</a>, so it can be understood be continuous integration
|
|
servers such as <a class="reference external" href="http://jenkins-ci.org">Jenkins</a>.</p>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="reference-part">
|
|
<h1><a class="toc-backref" href="#id13">Reference Part</a></h1>
|
|
<div class="section" id="cmd">
|
|
<h2><a class="toc-backref" href="#id14">cmd</a></h2>
|
|
<p>The <tt class="docutils literal">cmd</tt> method is the central method of the whole Cmdtest framework.
|
|
It should always be called with a block like this:</p>
|
|
<pre class="literal-block">
|
|
cmd "some_prog ..." do
|
|
assertion1 ...
|
|
...
|
|
assertionN ...
|
|
end
|
|
</pre>
|
|
<p>A block is used to make it easy to know when the last assertion has
|
|
been found. The do-block should only contain assertions. Cmdtest
|
|
applies some implicit assertions if the do-block is empty or misses
|
|
some kind of assertion:</p>
|
|
<pre class="literal-block">
|
|
# all assertions implicit
|
|
cmd "true" do
|
|
end
|
|
|
|
# exit status assertion explicit, but other assertions implicit
|
|
cmd "true" do
|
|
exit_zero
|
|
end
|
|
</pre>
|
|
<p>See also the example in the <a class="reference internal" href="#structure-of-a-test-method">Structure of a test-method</a> section above.</p>
|
|
</div>
|
|
<div class="section" id="assertions-exit-status">
|
|
<h2><a class="toc-backref" href="#id15">Assertions - exit status</a></h2>
|
|
<p>These methods should only be used inside a <tt class="docutils literal">cmd</tt> block.</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">exit_nonzero</tt></dt>
|
|
<dd>The command should have exited with a non-zero exit status (i.e. it
|
|
should have failed).</dd>
|
|
<dt><tt class="docutils literal">exit_status(status)</tt></dt>
|
|
<dd>The command should have exited with the specified exit status.</dd>
|
|
<dt><tt class="docutils literal">exit_zero</tt></dt>
|
|
<dd>The command should have exited with a zero exit status (i.e. it
|
|
should have succeeded). This is the default if none of the other
|
|
exit-related methods have been called.</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="assertions-files">
|
|
<h2><a class="toc-backref" href="#id16">Assertions - files</a></h2>
|
|
<p>These methods should only be used inside a <tt class="docutils literal">cmd</tt> block.</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal"><span class="pre">affected_files(file1,...,fileN)</span></tt></dt>
|
|
<dd>The specified files should have been created, removed or modified by the
|
|
command. This assertion can be used when it doesn't matter which
|
|
of <tt class="docutils literal">created_files</tt>, <tt class="docutils literal">removed_files</tt> or <tt class="docutils literal">changed_files</tt> that apply
|
|
(cf. <tt class="docutils literal">written_files</tt>).</dd>
|
|
<dt><tt class="docutils literal"><span class="pre">created_files(file1,...,fileN)</span></tt></dt>
|
|
<dd>The specified files should have been created by the command.</dd>
|
|
<dt><tt class="docutils literal"><span class="pre">modified_files(file1,...,fileN)</span></tt></dt>
|
|
<dd>The specified files should have been modified by the command. A
|
|
file is considered modified if it existed before the command, and
|
|
something about the file has changed after the command (inode
|
|
number, modification date or content).</dd>
|
|
<dt><tt class="docutils literal"><span class="pre">removed_files(file1,...,fileN)</span></tt></dt>
|
|
<dd>The specified files should have been removed by the command.</dd>
|
|
<dt><tt class="docutils literal"><span class="pre">written_files(file1,...,fileN)</span></tt></dt>
|
|
<dd>The specified files should have been created or modified by the
|
|
command. This assertion can be used when it doesn't matter which
|
|
of <tt class="docutils literal">created_files</tt> or <tt class="docutils literal">changed_files</tt> that apply. A typical scenario is
|
|
in a test method where repeated operations are done on the same
|
|
file. By using <tt class="docutils literal">written_files</tt> we don't have to treat the first
|
|
case special (when the file is created).</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="assertions-stdout-stderr-file-content">
|
|
<h2><a class="toc-backref" href="#id17">Assertions - stdout/stderr/file content</a></h2>
|
|
<p>These methods should only be used inside a <tt class="docutils literal">cmd</tt> block.</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">file_equal(file, content)</tt></dt>
|
|
<dd>Assert that the specified file matches the given content.
|
|
See "stdout_equal" for how "content" can be specified.</dd>
|
|
<dt><tt class="docutils literal">file_not_equal(file, content)</tt></dt>
|
|
<dd>Like <tt class="docutils literal">file_equal</tt> but with inverted test.</dd>
|
|
<dt><tt class="docutils literal">file_encoding(file, enc, bom: nil)</tt></dt>
|
|
<dd>Assert that the file uses encoding <tt class="docutils literal">enc</tt>. This is verified by reading
|
|
the file using that encoding. The optional <tt class="docutils literal">bom</tt> argument can be used
|
|
to assert the existence/non-existence of a Unicode BOM.</dd>
|
|
<dt><tt class="docutils literal">stderr_equal(content)</tt></dt>
|
|
<dd>Assert that the standard error of the command matches the given content.
|
|
See "stdout_equal" for how "content" can be specified.</dd>
|
|
<dt><tt class="docutils literal">stderr_not_equal(content)</tt></dt>
|
|
<dd>Like <tt class="docutils literal">stderr_equal</tt> but with inverted test.</dd>
|
|
<dt><tt class="docutils literal">stdout_equal(content)</tt></dt>
|
|
<dd>Assert that the standard output of the command matches the given content.
|
|
The content can be given in several different forms: 1) as a
|
|
string that should be equal to the entire file, 2) as an array of
|
|
lines that should be equal to the entire file, 3) as a regexp that
|
|
should match the entire file (given as one string).
|
|
For more details and examples see the section <a class="reference internal" href="#matching-standard-output-content">Matching standard output content</a>.</dd>
|
|
<dt><tt class="docutils literal">stdout_not_equal(content)</tt></dt>
|
|
<dd>Like <tt class="docutils literal">stdout_equal</tt> but with inverted test.</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="assertions-misc">
|
|
<h2><a class="toc-backref" href="#id18">Assertions - misc</a></h2>
|
|
<p>These methods should only be used inside a <tt class="docutils literal">cmd</tt> block.</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">assert(flag, msg=nil)</tt></dt>
|
|
<dd>Assert that <tt class="docutils literal">flag</tt> is true. This assertion is a last resort, when no other
|
|
assertion fits. Should normally not be used.</dd>
|
|
<dt><tt class="docutils literal">time(interval)</tt></dt>
|
|
<dd>Assert that executing the command took a number of seconds inside the
|
|
interval given as argument.</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="helper-functions">
|
|
<h2><a class="toc-backref" href="#id19">Helper functions</a></h2>
|
|
<p>These methods should only be used outside a <tt class="docutils literal">cmd</tt> block in a test method,
|
|
or in the <tt class="docutils literal">setup</tt> method.</p>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">create_file(filename, content)</tt></dt>
|
|
<dd>Create a file inside the "work directory".
|
|
If the filename contains a directory part, intermediate directories are
|
|
created if needed.
|
|
The content can be specified either as an array of lines or as
|
|
a string with the content of the whole file.
|
|
The filename is evaluated relative to the current directory at the
|
|
time of the call.</dd>
|
|
<dt><tt class="docutils literal">dont_ignore_files(file1, <span class="pre">...,</span> fileN)</tt></dt>
|
|
<dd>Don't ignore the specified files when looking for differences in the filesystem.
|
|
This overrides a previous call to <tt class="docutils literal">ignore_files</tt>.
|
|
If a previous call to <tt class="docutils literal">ignore_files</tt> ignored a whole directory tree, the call to
|
|
<tt class="docutils literal">dont_ignore_files</tt> can reverse the effect for specific files inside that
|
|
directory tree.</dd>
|
|
<dt><tt class="docutils literal">ignore_file(file)</tt></dt>
|
|
<dd>Ignore the specified file when looking for differences in the filesystem.
|
|
A subdirectory can be ignored by giving a trailing "/" to the name.</dd>
|
|
<dt><tt class="docutils literal">ignore_files(file1, <span class="pre">...,</span> fileN)</tt></dt>
|
|
<dd>Ignore the specified files when looking for differences in the filesystem.</dd>
|
|
<dt><tt class="docutils literal">import_file(src, tgt)</tt></dt>
|
|
<dd>Copy a file from outside of the "work directory" to inside.
|
|
The <tt class="docutils literal">src</tt> path is evaluated relative to the current directory
|
|
when <tt class="docutils literal">cmdtest</tt> was called. The <tt class="docutils literal">tgt</tt> is evaluated relative to
|
|
the current directory inside the "work directory" at the time
|
|
of the call.</dd>
|
|
<dt><tt class="docutils literal">import_directory(src, tgt)</tt></dt>
|
|
<dd>Copy a directory tree from outside of the "work directory" to inside.
|
|
The <tt class="docutils literal">src</tt> path is evaluated relative to the current directory
|
|
when <tt class="docutils literal">cmdtest</tt> was called. The <tt class="docutils literal">tgt</tt> is evaluated relative to
|
|
the current directory inside the "work directory" at the time
|
|
of the call. It is an error if <tt class="docutils literal">tgt</tt> already exists.</dd>
|
|
<dt><tt class="docutils literal">prepend_local_path(dir)</tt></dt>
|
|
<dd>Prepend the given directory to the <tt class="docutils literal">PATH</tt> so commands executed via <tt class="docutils literal">cmd</tt>
|
|
are looked up using the modified <tt class="docutils literal">PATH</tt>. The argument <tt class="docutils literal">dir</tt> is evaluated
|
|
relative to the current directory in effect at the time of the call
|
|
(i.e. typically the "work directory" during the test).</dd>
|
|
<dt><tt class="docutils literal">prepend_path(dir)</tt></dt>
|
|
<dd>Prepend the given directory to the <tt class="docutils literal">PATH</tt> so commands executed via <tt class="docutils literal">cmd</tt>
|
|
are looked up using the modified <tt class="docutils literal">PATH</tt>. A typical use is to add the directory
|
|
where the executable tested is located. The argument <tt class="docutils literal">dir</tt> is evaluated
|
|
relative to the current directory in effect when <tt class="docutils literal">cmdtest</tt> was invoked.</dd>
|
|
<dt><tt class="docutils literal">setenv(name, value)</tt></dt>
|
|
<dd>Set an environment variable that should be in effect when commands are executed
|
|
by later calls to <tt class="docutils literal">cmd</tt>.</dd>
|
|
<dt><tt class="docutils literal">unsetenv(name)</tt></dt>
|
|
<dd>Unset an environment variable that should not be in effect when commands are executed
|
|
by later calls to <tt class="docutils literal">cmd</tt>.</dd>
|
|
<dt><tt class="docutils literal">set_path(dir1, <span class="pre">...,</span> dirN)</tt></dt>
|
|
<dd>Set <tt class="docutils literal">PATH</tt> to the given directories, so commands executed via <tt class="docutils literal">cmd</tt>
|
|
are looked up using the modified <tt class="docutils literal">PATH</tt>. This method sets the whole <tt class="docutils literal">PATH</tt>
|
|
rather than modifying it (in contrast to <tt class="docutils literal">prepend_path</tt> and <tt class="docutils literal">prepend_local_path</tt>).</dd>
|
|
<dt><tt class="docutils literal">skip_test(reason)</tt></dt>
|
|
<dd>If a test method should not be run for some reason (eg. wrong platform),
|
|
this can be signaled to <tt class="docutils literal">cmdtest</tt> by calling
|
|
<tt class="docutils literal">skip_test</tt> at the beginning of the test method. The argument
|
|
should mention why the test is skipped. Such tests will be
|
|
reported as "skipped", and also show up in the JUnit format XML
|
|
files (the intention is that this should work like the "assume-functions"
|
|
in recent versions of JUnit).</dd>
|
|
<dt><tt class="docutils literal">touch_file(filename)</tt></dt>
|
|
<dd>"touch" a file inside the "work directory".
|
|
The filename is evaluated relative to the current directory at the
|
|
time of the call.</dd>
|
|
<dt><tt class="docutils literal">stdout_check()</tt></dt>
|
|
<dd><p class="first">Will "callback" to the test script, giving the user a chance to inpect STDOUT,
|
|
and maybe call "assert". Example:</p>
|
|
<pre class="last literal-block">
|
|
cmd "seq 0 5 100" do
|
|
stdout_check do |lines|
|
|
assert lines.include?(55), "no line '55'"
|
|
end
|
|
end
|
|
</pre>
|
|
</dd>
|
|
<dt><tt class="docutils literal">stdout_check()</tt></dt>
|
|
<dd>Will "callback" to the test script, giving the user a chance to inpect STDERR,
|
|
and maybe call "assert" (see <tt class="docutils literal">stdout_check()</tt> above).</dd>
|
|
</dl>
|
|
</div>
|
|
<div class="section" id="deprecated-helper-functions">
|
|
<h2><a class="toc-backref" href="#id20">Deprecated helper functions</a></h2>
|
|
<dl class="docutils">
|
|
<dt><tt class="docutils literal">dir_mkdir(file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">Dir.mkdir</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">file_chmod(arg, file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">File.chmod</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">file_open(file, *args, &block)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">File.open</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">file_read(file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">File.read</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">file_symlink(file1, file2)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">File.symlink</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">file_utime(arg1, arg2, file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">File.utime</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">remove_file(file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">FileUtils.rm_f</tt> instead.</dd>
|
|
<dt><tt class="docutils literal">remove_file_tree(file)</tt></dt>
|
|
<dd>Deprecated, use <tt class="docutils literal">FileUtils.rm_rf</tt> instead.</dd>
|
|
</dl>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="footer">
|
|
<hr class="footer" />
|
|
<a class="reference external" href="cmdtest.txt">View document source</a>.
|
|
Generated on: 2018-04-12.
|
|
Generated by <a class="reference external" href="http://docutils.sourceforge.net/">Docutils</a> from <a class="reference external" href="http://docutils.sourceforge.net/rst.html">reStructuredText</a> source.
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|