A Discrete-Event Network Simulator
API
test.cc
Go to the documentation of this file.
1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
2 /*
3  * Copyright (c) 2009 University of Washington
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as
7  * published by the Free Software Foundation;
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17  */
18 
19 #include "test.h"
20 #include "assert.h"
21 #include "abort.h"
22 #include "singleton.h"
23 #include "system-path.h"
24 #include "log.h"
25 #include "des-metrics.h"
26 #include <cmath>
27 #include <cstring>
28 #include <vector>
29 #include <list>
30 #include <map>
31 
32 
39 namespace ns3 {
40 
42 
43 bool
44 TestDoubleIsEqual (const double x1, const double x2, const double epsilon)
45 {
46  NS_LOG_FUNCTION (x1 << x2 << epsilon);
47  int exponent;
48  double delta, difference;
49 
50  //
51  // Find exponent of largest absolute value
52  //
53  {
54  double max = (std::fabs (x1) > std::fabs (x2)) ? x1 : x2;
55  (void)std::frexp (max, &exponent);
56  }
57 
58  //
59  // Form a neighborhood of size 2 * delta
60  //
61  delta = std::ldexp (epsilon, exponent);
62  difference = x1 - x2;
63 
64  if (difference > delta || difference < -delta)
65  {
66  return false;
67  }
68  return true;
69 }
70 
76 {
87  TestCaseFailure (std::string _cond, std::string _actual,
88  std::string _limit, std::string _message,
89  std::string _file, int32_t _line);
90  std::string cond;
91  std::string actual;
92  std::string limit;
93  std::string message;
94  std::string file;
95  int32_t line;
96 };
104 std::ostream & operator << (std::ostream & os, const TestCaseFailure & failure)
105 {
106  os << " test=\"" << failure.cond
107  << "\" actual=\"" << failure.actual
108  << "\" limit=\"" << failure.limit
109  << "\" in=\"" << failure.file
110  << ":" << failure.line
111  << "\" " << failure.message;
112 
113  return os;
114 }
115 
121 {
123  Result ();
124 
128  std::vector<TestCaseFailure> failure;
131 };
132 
138 class TestRunnerImpl : public Singleton<TestRunnerImpl>
139 {
140 public:
142  TestRunnerImpl ();
143 
148  void AddTestSuite (TestSuite *testSuite);
150  bool MustAssertOnFailure (void) const;
152  bool MustContinueOnFailure (void) const;
157  bool MustUpdateData (void) const;
166  std::string GetTopLevelSourceDir (void) const;
171  std::string GetTempDir (void) const;
173  int Run (int argc, char *argv[]);
174 
175 private:
176 
182  bool IsTopLevelSourceDir (std::string path) const;
202  std::string ReplaceXmlSpecialCharacters (std::string xml) const;
211  void PrintReport (TestCase *test, std::ostream *os, bool xml, int level);
219  void PrintTestNameList (std::list<TestCase *>::const_iterator begin,
220  std::list<TestCase *>::const_iterator end,
221  bool printTestType) const;
223  void PrintTestTypeList (void) const;
228  void PrintHelp (const char *programName) const;
240  std::list<TestCase *> FilterTests (std::string testName,
241  enum TestSuite::Type testType,
242  enum TestCase::TestDuration maximumTestDuration);
243 
244 
246  typedef std::vector<TestSuite *> TestSuiteVector;
247 
249  std::string m_tempDir;
250  bool m_verbose;
254 };
255 
256 
257 
258 TestCaseFailure::TestCaseFailure (std::string _cond, std::string _actual,
259  std::string _limit, std::string _message,
260  std::string _file, int32_t _line)
261  : cond (_cond), actual (_actual), limit (_limit),
262  message (_message), file (_file), line (_line)
263 {
264  NS_LOG_FUNCTION (this << _cond << _actual << _limit << _message << _file << _line);
265 }
267  : childrenFailed (false)
268 {
269  NS_LOG_FUNCTION (this);
270 }
271 
272 
273 
274 TestCase::TestCase (std::string name)
275  : m_parent (0),
276  m_dataDir (""),
277  m_runner (0),
278  m_result (0),
279  m_name (name),
281 {
282  NS_LOG_FUNCTION (this << name);
283 }
284 
286 {
287  NS_LOG_FUNCTION (this);
288  NS_ASSERT (m_runner == 0);
289  m_parent = 0;
290  delete m_result;
291  for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
292  {
293  delete *i;
294  }
295  m_children.clear ();
296 }
297 
298 void
300 {
301  NS_LOG_FUNCTION (&testCase << duration);
302 
303  // Test names are used to create temporary directories,
304  // so we test for illegal characters.
305  //
306  // Windows: <>:"/\|?*
307  // http://msdn.microsoft.com/en-us/library/aa365247(v=vs.85).aspx
308  // Mac: : (deprecated, was path separator in Mac OS Classic, pre X)
309  // Unix: / (and .. may give trouble?)
310  //
311  // The Windows list is too restrictive: we like to label
312  // tests with "val = v1 * v2" or "v1 < 3" or "case: foo --> bar"
313  // So we allow ':<>*"
314 
315  std::string badchars = "\"/\\|?";
316  // Badchar Class Regex Count of failing test names
317  // All ":<>\"/\\|?*" 611
318  // Allow ':' "<>\"/\\|?*" 128
319  // Allow ':<>' "\"/\\|?*" 12
320  // Allow ':<>*' "\"/\\|?" 0
321 
322  std::string::size_type badch = testCase->m_name.find_first_of (badchars);
323  if (badch != std::string::npos)
324  {
325  /*
326  To count the bad test names, use NS_LOG_UNCOND instead
327  of NS_FATAL_ERROR, and the command
328  $ ./waf --run "test-runner --list" 2>&1 | grep "^Invalid" | wc
329  */
330  NS_LOG_UNCOND ("Invalid test name: cannot contain any of '"
331  << badchars << "': " << testCase->m_name);
332  }
333 
334  testCase->m_duration = duration;
335  testCase->m_parent = this;
336  m_children.push_back (testCase);
337 }
338 
339 bool
340 TestCase::IsFailed (void) const
341 {
342  NS_LOG_FUNCTION (this);
343  return m_result->childrenFailed || !m_result->failure.empty ();
344 }
345 
346 void
348 {
349  NS_LOG_FUNCTION (this << runner);
350  m_result = new Result ();
351  m_runner = runner;
352  DoSetup ();
353  m_result->clock.Start ();
354  for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
355  {
356  TestCase *test = *i;
357  test->Run (runner);
358  if (IsFailed ())
359  {
360  goto out;
361  }
362  }
363  DoRun ();
364  out:
365  m_result->clock.End ();
366  DoTeardown ();
367  m_runner = 0;
368 }
369 std::string
370 TestCase::GetName (void) const
371 {
372  NS_LOG_FUNCTION (this);
373  return m_name;
374 }
375 TestCase *
377 {
378  return m_parent;
379 }
380 
381 void
382 TestCase::ReportTestFailure (std::string cond, std::string actual,
383  std::string limit, std::string message,
384  std::string file, int32_t line)
385 {
386  NS_LOG_FUNCTION (this << cond << actual << limit << message << file << line);
387  m_result->failure.push_back (TestCaseFailure (cond, actual, limit,
388  message, file, line));
389  // set childrenFailed flag on parents.
390  TestCase *current = m_parent;
391  while (current != 0)
392  {
393  current->m_result->childrenFailed = true;
394  current = current->m_parent;
395  }
396 
397 }
398 bool
400 {
401  NS_LOG_FUNCTION (this);
402  return m_runner->MustAssertOnFailure ();
403 }
404 bool
406 {
407  NS_LOG_FUNCTION (this);
408  return m_runner->MustContinueOnFailure ();
409 }
410 
411 std::string
412 TestCase::CreateDataDirFilename (std::string filename)
413 {
414  NS_LOG_FUNCTION (this << filename);
415  const TestCase *current = this;
416  while (current != 0 && current->m_dataDir == "")
417  {
418  current = current->m_parent;
419  }
420  if (current == 0)
421  {
422  NS_FATAL_ERROR ("No one called SetDataDir prior to calling this function");
423  }
424 
425  std::string a = SystemPath::Append (m_runner->GetTopLevelSourceDir (), current->m_dataDir);
426  std::string b = SystemPath::Append (a, filename);
427  return b;
428 }
429 std::string
430 TestCase::CreateTempDirFilename (std::string filename)
431 {
432  NS_LOG_FUNCTION (this << filename);
433  if (m_runner->MustUpdateData ())
434  {
435  return CreateDataDirFilename (filename);
436  }
437  else
438  {
439  std::list<std::string> names;
440  const TestCase *current = this;
441  while (current != 0)
442  {
443  names.push_front (current->m_name);
444  current = current->m_parent;
445  }
446  std::string tempDir = SystemPath::Append (m_runner->GetTempDir (), SystemPath::Join (names.begin (), names.end ()));
447  SystemPath::MakeDirectories (tempDir);
448  return SystemPath::Append (tempDir, filename);
449  }
450 }
451 bool
453 {
454  NS_LOG_FUNCTION (this);
455  return !IsStatusSuccess ();
456 }
457 bool
459 {
460  NS_LOG_FUNCTION (this);
461  return m_result->failure.empty ();
462 }
463 
464 void
465 TestCase::SetDataDir (std::string directory)
466 {
467  NS_LOG_FUNCTION (this << directory);
468  m_dataDir = directory;
469 }
470 
471 void
473 {
474  NS_LOG_FUNCTION (this);
475 }
476 void
478 {
479  NS_LOG_FUNCTION (this);
480 }
481 
482 
483 TestSuite::TestSuite (std::string name, TestSuite::Type type)
484  : TestCase (name),
485  m_type (type)
486 {
487  NS_LOG_FUNCTION (this << name << type);
489 }
490 
493 {
494  NS_LOG_FUNCTION (this);
495  return m_type;
496 }
497 
498 void
500 {
501  NS_LOG_FUNCTION (this);
502 }
503 
505  : m_tempDir (""),
506  m_assertOnFailure (false),
507  m_continueOnFailure (true),
508  m_updateData (false)
509 {
510  NS_LOG_FUNCTION (this);
511 }
512 
513 void
515 {
516  NS_LOG_FUNCTION (this << testSuite);
517  m_suites.push_back (testSuite);
518 }
519 
520 
521 bool
523 {
524  NS_LOG_FUNCTION (this);
525  return m_assertOnFailure;
526 }
527 bool
529 {
530  NS_LOG_FUNCTION (this);
531  return m_continueOnFailure;
532 }
533 
534 bool
536 {
537  NS_LOG_FUNCTION (this);
538  return m_updateData;
539 }
540 std::string
542 {
543  NS_LOG_FUNCTION (this);
544  return m_tempDir;
545 }
546 bool
547 TestRunnerImpl::IsTopLevelSourceDir (std::string path) const
548 {
549  NS_LOG_FUNCTION (this << path);
550  bool haveVersion = false;
551  bool haveLicense = false;
552 
553  //
554  // If there's a file named VERSION and a file named LICENSE in this
555  // directory, we assume it's our top level source directory.
556  //
557 
558  std::list<std::string> files = SystemPath::ReadFiles (path);
559  for (std::list<std::string>::const_iterator i = files.begin (); i != files.end (); ++i)
560  {
561  if (*i == "VERSION")
562  {
563  haveVersion = true;
564  }
565  else if (*i == "LICENSE")
566  {
567  haveLicense = true;
568  }
569  }
570 
571  return haveVersion && haveLicense;
572 }
573 
574 std::string
576 {
577  NS_LOG_FUNCTION (this);
578  std::string self = SystemPath::FindSelfDirectory ();
579  std::list<std::string> elements = SystemPath::Split (self);
580  while (!elements.empty ())
581  {
582  std::string path = SystemPath::Join (elements.begin (), elements.end ());
583  if (IsTopLevelSourceDir (path))
584  {
585  return path;
586  }
587  elements.pop_back ();
588  }
589  NS_FATAL_ERROR ("Could not find source directory from self=" << self);
590 }
591 
592 //
593 // XML files have restrictions on certain characters that may be present in
594 // data. We need to replace these characters with their alternate
595 // representation on the way into the XML file.
596 //
597 std::string
599 {
600  NS_LOG_FUNCTION (this << xml);
601  typedef std::map <char, std::string> specials_map;
602  specials_map specials;
603  specials['<'] = "&lt;";
604  specials['>'] = "&gt;";
605  specials['&'] = "&amp;";
606  specials['"'] = "&#39;";
607  specials['\''] = "&quot;";
608 
609  std::string result;
610  std::size_t length = xml.length ();
611 
612  for (size_t i = 0; i < length; ++i)
613  {
614  char character = xml[i];
615 
616  specials_map::const_iterator it = specials.find (character);
617 
618  if (it == specials.end ())
619  {
620  result.push_back (character);
621  }
622  else
623  {
624  result += it->second;
625  }
626  }
627  return result;
628 }
629 
631 struct Indent
632 {
637  Indent (int level);
639  int level;
640 };
641 Indent::Indent (int _level)
642  : level (_level)
643 {
644  NS_LOG_FUNCTION (this << _level);
645 }
652 std::ostream &operator << (std::ostream &os, const Indent &val)
653 {
654  for (int i = 0; i < val.level; i++)
655  {
656  os << " ";
657  }
658  return os;
659 }
660 
661 void
662 TestRunnerImpl::PrintReport (TestCase *test, std::ostream *os, bool xml, int level)
663 {
664  NS_LOG_FUNCTION (this << test << os << xml << level);
665  if (test->m_result == 0)
666  {
667  // Do not print reports for tests that were not run.
668  return;
669  }
670  // Report times in seconds, from ms timer
671  const double MS_PER_SEC = 1000.;
672  double real = test->m_result->clock.GetElapsedReal () / MS_PER_SEC;
673  double user = test->m_result->clock.GetElapsedUser () / MS_PER_SEC;
674  double system = test->m_result->clock.GetElapsedSystem () / MS_PER_SEC;
675 
676  std::streamsize oldPrecision = (*os).precision (3);
677  *os << std::fixed;
678 
679  std::string statusString = test->IsFailed ()?"FAIL":"PASS";
680  if (xml)
681  {
682  *os << Indent (level) << "<Test>" << std::endl;
683  *os << Indent (level+1) << "<Name>" << ReplaceXmlSpecialCharacters (test->m_name)
684  << "</Name>" << std::endl;
685  *os << Indent (level+1) << "<Result>" << statusString << "</Result>" << std::endl;
686  *os << Indent (level+1) << "<Time real=\"" << real << "\" user=\"" << user
687  << "\" system=\"" << system << "\"/>" << std::endl;
688  for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
689  {
690  TestCaseFailure failure = test->m_result->failure[i];
691  *os << Indent (level+2) << "<FailureDetails>" << std::endl
692  << Indent (level+3) << "<Condition>"
693  << ReplaceXmlSpecialCharacters (failure.cond) << "</Condition>" << std::endl
694  << Indent (level+3) << "<Actual>"
695  << ReplaceXmlSpecialCharacters (failure.actual) << "</Actual>" << std::endl
696  << Indent (level+3) << "<Limit>"
697  << ReplaceXmlSpecialCharacters (failure.limit) << "</Limit>" << std::endl
698  << Indent (level+3) << "<Message>"
699  << ReplaceXmlSpecialCharacters (failure.message) << "</Message>" << std::endl
700  << Indent (level+3) << "<File>"
701  << ReplaceXmlSpecialCharacters (failure.file) << "</File>" << std::endl
702  << Indent (level+3) << "<Line>" << failure.line << "</Line>" << std::endl
703  << Indent (level+2) << "</FailureDetails>" << std::endl;
704  }
705  for (uint32_t i = 0; i < test->m_children.size (); i++)
706  {
707  TestCase *child = test->m_children[i];
708  PrintReport (child, os, xml, level + 1);
709  }
710  *os << Indent (level) << "</Test>" << std::endl;
711  }
712  else
713  {
714  *os << Indent (level) << statusString << " " << test->GetName ()
715  << " " << real << " s" << std::endl;
716  if (m_verbose)
717  {
718  for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
719  {
720  *os << Indent (level) << test->m_result->failure[i] << std::endl;
721  }
722  for (uint32_t i = 0; i < test->m_children.size (); i++)
723  {
724  TestCase *child = test->m_children[i];
725  PrintReport (child, os, xml, level + 1);
726  }
727  }
728  }
729 
730  (*os).unsetf(std::ios_base::floatfield);
731  (*os).precision (oldPrecision);
732 }
733 
734 void
735 TestRunnerImpl::PrintHelp (const char *program_name) const
736 {
737  NS_LOG_FUNCTION (this << program_name);
738  std::cout << "Usage: " << program_name << " [OPTIONS]" << std::endl
739  << std::endl
740  << "Options: " << std::endl
741  << " --help : print these options" << std::endl
742  << " --print-test-name-list : print the list of names of tests available" << std::endl
743  << " --list : an alias for --print-test-name-list" << std::endl
744  << " --print-test-types : print the type of tests along with their names" << std::endl
745  << " --print-test-type-list : print the list of types of tests available" << std::endl
746  << " --print-temp-dir : print name of temporary directory before running " << std::endl
747  << " the tests" << std::endl
748  << " --test-type=TYPE : process only tests of type TYPE" << std::endl
749  << " --test-name=NAME : process only test whose name matches NAME" << std::endl
750  << " --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
751  << " for --test-name=NAME" << std::endl
752  << " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
753  << " when running under a debugger" << std::endl
754  << " --stop-on-failure : when a test fails, stop immediately" << std::endl
755  << " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
756  << " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
757  << " includes QUICK and TAKES_FOREVER includes " << std::endl
758  << " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
759  << " run by default)" << std::endl
760  << " --verbose : print details of test execution" << std::endl
761  << " --xml : format test run output as xml" << std::endl
762  << " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
763  << " --datadir=DIR : set data dir for tests to read reference files" << std::endl
764  << " --out=FILE : send test result to FILE instead of standard "
765  << "output" << std::endl
766  << " --append=FILE : append test result to FILE instead of standard "
767  << "output" << std::endl
768  ;
769 }
770 
771 void
772 TestRunnerImpl::PrintTestNameList (std::list<TestCase *>::const_iterator begin,
773  std::list<TestCase *>::const_iterator end,
774  bool printTestType) const
775 {
776  NS_LOG_FUNCTION (this << &begin << &end << printTestType);
777  std::map<TestSuite::Type, std::string> label;
778 
779  label[TestSuite::ALL] = "all ";
780  label[TestSuite::BVT] = "bvt ";
781  label[TestSuite::UNIT] = "unit ";
782  label[TestSuite::SYSTEM] = "system ";
783  label[TestSuite::EXAMPLE] = "example ";
784  label[TestSuite::PERFORMANCE] = "performance ";
785 
786  for (std::list<TestCase *>::const_iterator i = begin; i != end; ++i)
787  {
788  TestSuite * test= dynamic_cast<TestSuite *>(*i);
789  NS_ASSERT (test != 0);
790  if (printTestType)
791  {
792  std::cout << label[test->GetTestType ()];
793  }
794  std::cout << test->GetName () << std::endl;
795  }
796 }
797 
798 void
800 {
801  NS_LOG_FUNCTION (this);
802  std::cout << " bvt: Build Verification Tests (to see if build completed successfully)" << std::endl;
803  std::cout << " core: Run all TestSuite-based tests (exclude examples)" << std::endl;
804  std::cout << " example: Examples (to see if example programs run successfully)" << std::endl;
805  std::cout << " performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
806  std::cout << " system: System Tests (spans modules to check integration of modules)" << std::endl;
807  std::cout << " unit: Unit Tests (within modules to check basic functionality)" << std::endl;
808 }
809 
810 
811 std::list<TestCase *>
812 TestRunnerImpl::FilterTests (std::string testName,
813  enum TestSuite::Type testType,
814  enum TestCase::TestDuration maximumTestDuration)
815 {
816  NS_LOG_FUNCTION (this << testName << testType);
817  std::list<TestCase *> tests;
818  for (uint32_t i = 0; i < m_suites.size (); ++i)
819  {
820  TestSuite *test = m_suites[i];
821  if (testType != TestSuite::ALL && test->GetTestType () != testType)
822  {
823  // skip test
824  continue;
825  }
826  if (testName != "" && test->GetName () != testName)
827  {
828  // skip test
829  continue;
830  }
831 
832  // Remove any test cases that should be skipped.
833  std::vector<TestCase *>::iterator j;
834  for (j = test->m_children.begin (); j != test->m_children.end ();)
835  {
836  TestCase *testCase = *j;
837 
838  // If this test case takes longer than the maximum test
839  // duration that should be run, then don't run it.
840  if (testCase->m_duration > maximumTestDuration)
841  {
842  // Free this test case's memory.
843  delete *j;
844 
845  // Remove this test case from the test suite.
846  j = test->m_children.erase (j);
847  }
848  else
849  {
850  // Only advance through the vector elements if this test
851  // case wasn't deleted.
852  ++j;
853  }
854  }
855 
856  // Add this test suite.
857  tests.push_back (test);
858  }
859  return tests;
860 }
861 
862 
863 int
864 TestRunnerImpl::Run (int argc, char *argv[])
865 {
866  NS_LOG_FUNCTION (this << argc << argv);
867  std::string testName = "";
868  std::string testTypeString = "";
869  std::string out = "";
870  std::string fullness = "";
871  bool xml = false;
872  bool append = false;
873  bool printTempDir = false;
874  bool printTestTypeList = false;
875  bool printTestNameList = false;
876  bool printTestTypeAndName = false;
877  enum TestCase::TestDuration maximumTestDuration = TestCase::QUICK;
878  char *progname = argv[0];
879 
880  char ** argi = argv;
881  ++argi;
882 
883  while (*argi != 0)
884  {
885  char *arg = *argi;
886 
887  if (strcmp(arg, "--assert-on-failure") == 0)
888  {
889  m_assertOnFailure = true;
890  }
891  else if (strcmp (arg, "--stop-on-failure") == 0)
892  {
893  m_continueOnFailure = false;
894  }
895  else if (strcmp (arg, "--verbose") == 0)
896  {
897  m_verbose = true;
898  }
899  else if (strcmp (arg, "--print-temp-dir") == 0)
900  {
901  printTempDir = true;
902  }
903  else if (strcmp (arg, "--update-data") == 0)
904  {
905  m_updateData = true;
906  }
907  else if (strcmp (arg, "--help") == 0)
908  {
909  PrintHelp (progname);
910  return 0;
911  }
912  else if (strcmp (arg, "--print-test-name-list") == 0 ||
913  strcmp(arg, "--list") == 0)
914  {
915  printTestNameList = true;
916  }
917  else if (strcmp (arg, "--print-test-types") == 0)
918  {
919  printTestTypeAndName = true;
920  }
921  else if (strcmp (arg, "--print-test-type-list") == 0)
922  {
923  printTestTypeList = true;
924  }
925  else if (strcmp(arg, "--append") == 0)
926  {
927  append = true;
928  }
929  else if (strcmp(arg, "--xml") == 0)
930  {
931  xml = true;
932  }
933  else if (strncmp(arg, "--test-type=", strlen("--test-type=")) == 0)
934  {
935  testTypeString = arg + strlen("--test-type=");
936  }
937  else if (strncmp(arg, "--test-name=", strlen("--test-name=")) == 0)
938  {
939  testName = arg + strlen("--test-name=");
940  }
941  else if (strncmp(arg, "--suite=", strlen("--suite=")) == 0)
942  {
943  testName = arg + strlen("--suite=");
944  }
945  else if (strncmp(arg, "--tempdir=", strlen("--tempdir=")) == 0)
946  {
947  m_tempDir = arg + strlen("--tempdir=");
948  }
949  else if (strncmp(arg, "--out=", strlen("--out=")) == 0)
950  {
951  out = arg + strlen("--out=");
952  }
953  else if (strncmp(arg, "--fullness=", strlen("--fullness=")) == 0)
954  {
955  fullness = arg + strlen("--fullness=");
956 
957  // Set the maximum test length allowed.
958  if (fullness == "QUICK")
959  {
960  maximumTestDuration = TestCase::QUICK;
961  }
962  else if (fullness == "EXTENSIVE")
963  {
964  maximumTestDuration = TestCase::EXTENSIVE;
965  }
966  else if (fullness == "TAKES_FOREVER")
967  {
968  maximumTestDuration = TestCase::TAKES_FOREVER;
969  }
970  else
971  {
972  // Wrong fullness option
973  PrintHelp (progname);
974  return 3;
975  }
976  }
977  else
978  {
979  // un-recognized command-line argument
980  PrintHelp (progname);
981  return 0;
982  }
983  argi++;
984  }
985  enum TestSuite::Type testType;
986  if (testTypeString == "")
987  {
988  testType = TestSuite::ALL;
989  }
990  else if (testTypeString == "bvt")
991  {
992  testType = TestSuite::BVT;
993  }
994  else if (testTypeString == "core")
995  {
996  testType = TestSuite::ALL;
997  }
998  else if (testTypeString == "example")
999  {
1000  testType = TestSuite::EXAMPLE;
1001  }
1002  else if (testTypeString == "unit")
1003  {
1004  testType = TestSuite::UNIT;
1005  }
1006  else if (testTypeString == "system")
1007  {
1008  testType = TestSuite::SYSTEM;
1009  }
1010  else if (testTypeString == "performance")
1011  {
1012  testType = TestSuite::PERFORMANCE;
1013  }
1014  else
1015  {
1016  std::cout << "Invalid test type specified: " << testTypeString << std::endl;
1017  PrintTestTypeList ();
1018  return 1;
1019  }
1020 
1021  std::list<TestCase *> tests = FilterTests (testName, testType, maximumTestDuration);
1022 
1023  if (m_tempDir == "")
1024  {
1026  }
1027  if (printTempDir)
1028  {
1029  std::cout << m_tempDir << std::endl;
1030  }
1031  if (printTestNameList)
1032  {
1033  PrintTestNameList (tests.begin (), tests.end (), printTestTypeAndName);
1034  return 0;
1035  }
1036  if (printTestTypeList)
1037  {
1038  PrintTestTypeList ();
1039  return 0;
1040  }
1041 
1042 
1043  std::ostream *os;
1044  if (out != "")
1045  {
1046  std::ofstream *ofs;
1047  ofs = new std::ofstream();
1048  std::ios_base::openmode mode = std::ios_base::out;
1049  if (append)
1050  {
1051  mode |= std::ios_base::app;
1052  }
1053  else
1054  {
1055  mode |= std::ios_base::trunc;
1056  }
1057  ofs->open (out.c_str (), mode);
1058  os = ofs;
1059  }
1060  else
1061  {
1062  os = &std::cout;
1063  }
1064 
1065  // let's run our tests now.
1066  bool failed = false;
1067  if (tests.size () == 0)
1068  {
1069  std::cerr << "Error: no tests match the requested string" << std::endl;
1070  return 1;
1071  }
1072  for (std::list<TestCase *>::const_iterator i = tests.begin (); i != tests.end (); ++i)
1073  {
1074  TestCase *test = *i;
1075 
1076 #ifdef ENABLE_DES_METRICS
1077  {
1078  /*
1079  Reorganize argv
1080  Since DES Metrics uses argv[0] for the trace file name,
1081  grab the test name and put it in argv[0],
1082  with test-runner as argv[1]
1083  then the rest of the original arguments.
1084  */
1085  std::string testname = test->GetName ();
1086  std::string runner = "[" + SystemPath::Split (argv[0]).back () + "]";
1087 
1088  std::vector<std::string> desargs;
1089  desargs.push_back (testname);
1090  desargs.push_back (runner);
1091  for (int i = 1; i < argc; ++i)
1092  {
1093  desargs.push_back (argv[i]);
1094  }
1095 
1096  DesMetrics::Get ()->Initialize (desargs, m_tempDir);
1097  }
1098 #endif
1099 
1100  test->Run (this);
1101  PrintReport (test, os, xml, 0);
1102  if (test->IsFailed ())
1103  {
1104  failed = true;
1105  if (!m_continueOnFailure)
1106  {
1107  return 1;
1108  }
1109  }
1110  }
1111 
1112  if (out != "")
1113  {
1114  delete os;
1115  }
1116 
1117  return failed?1:0;
1118 }
1119 
1120 int
1121 TestRunner::Run (int argc, char *argv[])
1122 {
1123  NS_LOG_FUNCTION (argc << argv);
1124  return TestRunnerImpl::Get ()->Run (argc, argv);
1125 }
1126 
1127 } // namespace ns3
std::string message
The associated message.
Definition: test.cc:93
virtual void DoSetup(void)
Implementation to do any local setup required for this TestCase.
Definition: test.cc:472
Helper to indent output a specified number of steps.
Definition: test.cc:631
virtual void DoTeardown(void)
Implementation to do any local setup required for this TestCase.
Definition: test.cc:477
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by "...
ns3::Singleton declaration and template implementation.
This test suite implements a System Test.
Definition: test.h:1353
Container for all tests.
Definition: test.cc:138
bool IsStatusFailure(void) const
Check if any tests failed.
Definition: test.cc:452
TestCaseFailure(std::string _cond, std::string _actual, std::string _limit, std::string _message, std::string _file, int32_t _line)
Constructor.
Definition: test.cc:258
SystemWallClockMs clock
Test running time.
Definition: test.cc:126
A suite of tests to run.
Definition: test.h:1342
Very long running test.
Definition: test.h:1162
Container for results from a TestCase.
Definition: test.cc:120
void AddTestSuite(TestSuite *testSuite)
Add a new top-level TestSuite.
Definition: test.cc:514
#define NS_ASSERT(condition)
At runtime, in debugging builds, if this condition is not true, the program prints the source file...
Definition: assert.h:67
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition: log.h:202
Type
Type of test.
Definition: test.h:1349
bool m_continueOnFailure
true if we should continue on failure.
Definition: test.cc:252
bool MustAssertOnFailure(void) const
Check if this run should assert on failure.
Definition: test.cc:399
#define NS_FATAL_ERROR(msg)
Report a fatal error with a message and terminate.
Definition: fatal-error.h:162
ns3::TestCase, ns3::TestSuite, ns3::TestRunner declarations, and NS_TEST_ASSERT macro definitions...
encapsulates test code
Definition: test.h:1155
enum TestDuration m_duration
TestCase duration.
Definition: test.h:1332
static TestRunnerImpl * Get(void)
Get a pointer to the singleton instance.
Definition: singleton.h:89
TestSuiteVector m_suites
The list of tests.
Definition: test.cc:248
void PrintReport(TestCase *test, std::ostream *os, bool xml, int level)
Print the test report.
Definition: test.cc:662
ns3::DesMetrics declaration.
TestSuite(std::string name, Type type=UNIT)
Construct a new test suite.
Definition: test.cc:483
std::list< std::string > Split(std::string path)
Split a file system path into directories according to the local path separator.
Definition: system-path.cc:205
bool MustContinueOnFailure(void) const
Check if this run should continue on failure.
Definition: test.cc:528
bool MustAssertOnFailure(void) const
Check if this run should assert on failure.
Definition: test.cc:522
virtual void DoRun(void)
Implementation to actually run this TestCase.
Definition: test.cc:499
bool IsStatusSuccess(void) const
Check if all tests passed.
Definition: test.cc:458
std::string GetTempDir(void) const
Get the path to temporary directory.
Definition: test.cc:541
std::list< TestCase * > FilterTests(std::string testName, enum TestSuite::Type testType, enum TestCase::TestDuration maximumTestDuration)
Generate the list of tests matching the constraints.
Definition: test.cc:812
ns3::SystemPath declarations.
A template singleton.
Definition: singleton.h:63
TestRunnerImpl()
Constructor.
Definition: test.cc:504
void Initialize(std::vector< std::string > args, std::string outDir="")
Open the DesMetrics trace file and print the header.
Definition: des-metrics.cc:42
Measure elapsed wall clock time in milliseconds.
std::string Join(std::list< std::string >::const_iterator begin, std::list< std::string >::const_iterator end)
Join a list of file system path directories into a single file system path.
Definition: system-path.cc:223
TestCase * m_parent
Pointer to my parent TestCase.
Definition: test.h:1324
void PrintTestTypeList(void) const
Print the list of test types.
Definition: test.cc:799
TestRunnerImpl * m_runner
Pointer to the TestRunner.
Definition: test.h:1329
#define max(a, b)
Definition: 80211b.c:43
void AddTestCase(TestCase *testCase, TestDuration duration=QUICK)
Add an individual child TestCase to this test suite.
Definition: test.cc:299
std::string cond
The name of the condition being tested.
Definition: test.cc:90
NS_ASSERT() and NS_ASSERT_MSG() macro definitions.
std::string MakeTemporaryDirectoryName(void)
Get the name of a temporary directory.
Definition: system-path.cc:281
bool IsTopLevelSourceDir(std::string path) const
Check if this is the root of the source tree.
Definition: test.cc:547
std::list< std::string > ReadFiles(std::string path)
Get the list of files located in a file system directory.
Definition: system-path.cc:242
Medium length test.
Definition: test.h:1161
bool childrenFailed
true if any child TestCases failed.
Definition: test.cc:130
bool TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
Compare two double precision floating point numbers and declare them equal if they are within some ep...
Definition: test.cc:44
std::string m_tempDir
The temporary directory.
Definition: test.cc:249
void Start(void)
Start a measure.
TestSuite::Type m_type
Type of this TestSuite.
Definition: test.h:1377
std::string GetTopLevelSourceDir(void) const
Get the path to the root of the source tree.
Definition: test.cc:575
This test suite implements a Performance Test.
Definition: test.h:1355
TestCase(std::string name)
Constructor.
Definition: test.cc:274
This test suite implements a Build Verification Test.
Definition: test.h:1351
std::ostream & operator<<(std::ostream &os, const Angles &a)
print a struct Angles to output
Definition: angles.cc:42
TestDuration
How long the test takes to execute.
Definition: test.h:1159
virtual ~TestCase()
Destructor.
Definition: test.cc:285
bool m_updateData
true if we should update reference data.
Definition: test.cc:253
int32_t line
The source line.
Definition: test.cc:95
Every class exported by the ns3 library is enclosed in the ns3 namespace.
std::string CreateDataDirFilename(std::string filename)
Construct the full path to a file in the data directory.
Definition: test.cc:412
virtual void DoRun(void)=0
Implementation to actually run this TestCase.
struct Result * m_result
Results data.
Definition: test.h:1330
void ReportTestFailure(std::string cond, std::string actual, std::string limit, std::string message, std::string file, int32_t line)
Log the failure of this TestCase.
Definition: test.cc:382
TestCase * GetParent() const
Get the parent of this TestCsse.
Definition: test.cc:376
#define NS_LOG_UNCOND(msg)
Output the requested message unconditionally.
bool m_verbose
Produce verbose output.
Definition: test.cc:250
void Run(TestRunnerImpl *runner)
Actually run this TestCase.
Definition: test.cc:347
void MakeDirectories(std::string path)
Create all the directories leading to path.
Definition: system-path.cc:328
Indent(int level)
Constructor.
Definition: test.cc:641
bool MustContinueOnFailure(void) const
Check if this run should continue on failure.
Definition: test.cc:405
Fast test.
Definition: test.h:1160
std::string ReplaceXmlSpecialCharacters(std::string xml) const
Clean up characters not allowed in XML.
Definition: test.cc:598
std::string actual
The actual value returned by the test.
Definition: test.cc:91
std::string FindSelfDirectory(void)
Get the file system path to the current executable.
Definition: system-path.cc:97
std::string GetName(void) const
Definition: test.cc:370
std::string file
The source file.
Definition: test.cc:94
std::string CreateTempDirFilename(std::string filename)
Construct the full path to a file in a temporary directory.
Definition: test.cc:430
std::string Append(std::string left, std::string right)
Join two file system path elements.
Definition: system-path.cc:188
Result()
Constructor.
Definition: test.cc:266
void SetDataDir(std::string directory)
Set the data directory where reference trace files can be found.
Definition: test.cc:465
std::vector< TestSuite * > TestSuiteVector
Container type for the test.
Definition: test.cc:246
std::vector< TestCase * > m_children
Vector of my children.
Definition: test.h:1327
This test suite implements an Example Test.
Definition: test.h:1354
bool MustUpdateData(void) const
Check if this run should update the reference data.
Definition: test.cc:535
Container for details of a test failure.
Definition: test.cc:75
int level
The number of steps.
Definition: test.cc:639
This test suite implements a Unit Test.
Definition: test.h:1352
TestSuite::Type GetTestType(void)
get the kind of test this test suite implements
Definition: test.cc:492
std::vector< TestCaseFailure > failure
TestCaseFailure records for each child.
Definition: test.cc:128
Debug message logging.
int64_t End(void)
Stop measuring the time since Start() was called.
static int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition: test.cc:1121
int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments. ...
Definition: test.cc:864
void PrintTestNameList(std::list< TestCase *>::const_iterator begin, std::list< TestCase *>::const_iterator end, bool printTestType) const
Print the list of all requested test suites.
Definition: test.cc:772
std::string m_name
TestCase name.
Definition: test.h:1331
bool IsFailed(void) const
Check if any tests failed.
Definition: test.cc:340
std::string m_dataDir
My data directory.
Definition: test.h:1328
bool m_assertOnFailure
true if we should assert on failure.
Definition: test.cc:251
void PrintHelp(const char *programName) const
Print the help text.
Definition: test.cc:735
NS_ABORT_x macro definitions.
std::string limit
The expected value.
Definition: test.cc:92