Soltest error handling refactor

libevmasm/Assembly.cpp

@@ -43,9 +43,9 @@
 #include <range/v3/view/enumerate.hpp>
 #include <range/v3/view/map.hpp>
 
-#include <fstream>
 #include <limits>
 #include <iterator>
+#include <ostream>
 #include <stack>
 
 using namespace solidity;

libevmasm/EVMAssemblyStack.cpp

@@ -74,13 +74,13 @@ void EVMAssemblyStack::assemble()
 LinkerObject const& EVMAssemblyStack::object(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);
-	return m_object;
+	return object();
 }
 
 LinkerObject const& EVMAssemblyStack::runtimeObject(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);
-	return m_runtimeObject;
+	return runtimeObject();
 }
 
 std::map<std::string, unsigned> EVMAssemblyStack::sourceIndices() const
@@ -95,13 +95,13 @@ std::map<std::string, unsigned> EVMAssemblyStack::sourceIndices() const
 std::string const* EVMAssemblyStack::sourceMapping(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);
-	return &m_sourceMapping;
+	return &sourceMapping();
 }
 
 std::string const* EVMAssemblyStack::runtimeSourceMapping(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);
-	return &m_runtimeSourceMapping;
+	return &runtimeSourceMapping();
 }
 
 Json EVMAssemblyStack::ethdebug(std::string const& _contractName) const
@@ -123,20 +123,30 @@ Json EVMAssemblyStack::ethdebug() const
 	return {};
 }
 
-Json EVMAssemblyStack::assemblyJSON(std::string const& _contractName) const
+Json EVMAssemblyStack::assemblyJSON() const
 {
-	solAssert(_contractName == m_name);
 	solAssert(m_evmAssembly);
 	return m_evmAssembly->assemblyJSON(sourceIndices());
 }
 
-std::string EVMAssemblyStack::assemblyString(std::string const& _contractName, StringMap const& _sourceCodes) const
+Json EVMAssemblyStack::assemblyJSON(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);
+	return assemblyJSON();
+}
+
+std::string EVMAssemblyStack::assemblyString(StringMap const& _sourceCodes) const
+{
 	solAssert(m_evmAssembly);
 	return m_evmAssembly->assemblyString(m_debugInfoSelection, _sourceCodes);
 }
 
+std::string EVMAssemblyStack::assemblyString(std::string const& _contractName, StringMap const& _sourceCodes) const
+{
+	solAssert(_contractName == m_name);
+	return assemblyString(_sourceCodes);
+}
+
 std::string const EVMAssemblyStack::filesystemFriendlyName(std::string const& _contractName) const
 {
 	solAssert(_contractName == m_name);

libevmasm/EVMAssemblyStack.h

@@ -58,20 +58,26 @@ class EVMAssemblyStack: public AbstractAssemblyStack
 
 	std::string const& name() const { return m_name; }
 
+	LinkerObject const& object() const { return m_object; }
 	LinkerObject const& object(std::string const& _contractName) const override;
+	LinkerObject const& runtimeObject() const { return m_runtimeObject; }
 	LinkerObject const& runtimeObject(std::string const& _contractName) const override;
 
 	std::shared_ptr<evmasm::Assembly> const& evmAssembly() const { return m_evmAssembly; }
 	std::shared_ptr<evmasm::Assembly> const& evmRuntimeAssembly() const { return m_evmRuntimeAssembly; }
 
+	std::string const& sourceMapping() const { return m_sourceMapping; }
 	std::string const* sourceMapping(std::string const& _contractName) const override;
+	std::string const& runtimeSourceMapping() const { return m_runtimeSourceMapping; }
 	std::string const* runtimeSourceMapping(std::string const& _contractName) const override;
 
 	Json ethdebug(std::string const& _contractName) const override;
 	Json ethdebugRuntime(std::string const& _contractName) const override;
 	Json ethdebug() const override;
 
+	Json assemblyJSON() const;
 	Json assemblyJSON(std::string const& _contractName) const override;
+	std::string assemblyString(StringMap const& _sourceCodes) const;
 	std::string assemblyString(std::string const& _contractName, StringMap const& _sourceCodes) const override;
 
 	std::string const filesystemFriendlyName(std::string const& _contractName) const override;

libevmasm/Instruction.cpp

@@ -26,7 +26,7 @@ using namespace solidity;
 using namespace solidity::util;
 using namespace solidity::evmasm;
 
-std::map<std::string, Instruction> const solidity::evmasm::c_instructions =
+std::map<std::string, Instruction, std::less<>> const solidity::evmasm::c_instructions =
 {
 	{ "STOP", Instruction::STOP },
 	{ "ADD", Instruction::ADD },

libevmasm/Instruction.h

@@ -322,6 +322,6 @@ InstructionInfo instructionInfo(Instruction _inst, langutil::EVMVersion _evmVers
 bool isValidInstruction(Instruction _inst);
 
 /// Convert from string mnemonic to Instruction type.
-extern const std::map<std::string, Instruction> c_instructions;
+extern const std::map<std::string, Instruction, std::less<>> c_instructions;
 
 }

test/CMakeLists.txt

@@ -51,7 +51,11 @@ detect_stray_source_files("${libsolutil_sources}" "libsolutil/")
 
 set(libevmasm_sources
     libevmasm/Assembler.cpp
+    libevmasm/EVMAssemblyTest.cpp
+    libevmasm/EVMAssemblyTest.h
     libevmasm/Optimiser.cpp
+    libevmasm/PlainAssemblyParser.cpp
+    libevmasm/PlainAssemblyParser.h
 )
 detect_stray_source_files("${libevmasm_sources}" "libevmasm/")
 

test/Common.cpp

@@ -24,6 +24,7 @@
 #include <libyul/backends/evm/EVMDialect.h>
 
 #include <libsolutil/Assertions.h>
+#include <libsolutil/Exceptions.h>
 #include <libsolutil/StringUtils.h>
 
 #include <boost/algorithm/string.hpp>
@@ -33,7 +34,6 @@
 #include <range/v3/all.hpp>
 
 #include <iostream>
-#include <stdexcept>
 
 namespace fs = boost::filesystem;
 namespace po = boost::program_options;
@@ -110,10 +110,10 @@ CommonOptions::CommonOptions(std::string _caption):
 void CommonOptions::addOptions()
 {
 	options.add_options()
-		("evm-version", po::value(&evmVersionString), "which EVM version to use")
+		("evm-version", po::value(&m_evmVersionString), "which EVM version to use")
 		// "eof-version" is declared as uint64_t, since uint8_t will be parsed as character by boost.
 		("eof-version", po::value<uint64_t>()->implicit_value(1u), "which EOF version to use")
-		("testpath", po::value<fs::path>(&this->testPath)->default_value(solidity::test::testPath()), "path to test files")
+		("testpath", po::value<fs::path>(&this->testPath)->default_value(test::testPath()), "path to test files")
 		("vm", po::value<std::vector<fs::path>>(&vmPaths), "path to evmc library, can be supplied multiple times.")
 		("batches", po::value<size_t>(&this->batches)->default_value(1), "set number of batches to split the tests into")
 		("selected-batch", po::value<size_t>(&this->selectedBatch)->default_value(0), "zero-based number of batch to execute")
@@ -185,8 +185,7 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 		{
 			// Request as uint64_t, since uint8_t will be parsed as character by boost.
 			uint64_t eofVersion = arguments["eof-version"].as<uint64_t>();
-			if (eofVersion != 1)
-				BOOST_THROW_EXCEPTION(std::runtime_error("Invalid EOF version: " + std::to_string(eofVersion)));
+			solRequire(eofVersion == 1, ConfigException, "Invalid EOF version: " + std::to_string(eofVersion));
 			m_eofVersion = 1;
 		}
 
@@ -198,11 +197,7 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 					(parsedOption.original_tokens.size() == 1 && parsedOption.original_tokens.front().empty())
 				)
 					continue; // ignore empty options
-				std::stringstream errorMessage;
-				errorMessage << "Unrecognized option: ";
-				for (auto const& token: parsedOption.original_tokens)
-					errorMessage << token;
-				BOOST_THROW_EXCEPTION(std::runtime_error(errorMessage.str()));
+				solThrow(ConfigException, "Unrecognized option: " + util::joinHumanReadable(parsedOption.original_tokens, ""));
 			}
 	}
 	catch (po::error const& exception)
@@ -259,11 +254,10 @@ void CommonOptions::printSelectedOptions(std::ostream& _stream, std::string cons
 
 langutil::EVMVersion CommonOptions::evmVersion() const
 {
-	if (!evmVersionString.empty())
+	if (!m_evmVersionString.empty())
 	{
-		auto version = langutil::EVMVersion::fromString(evmVersionString);
-		if (!version)
-			BOOST_THROW_EXCEPTION(std::runtime_error("Invalid EVM version: " + evmVersionString));
+		auto version = langutil::EVMVersion::fromString(m_evmVersionString);
+		solRequire(version, ConfigException, "Invalid EVM version: " + m_evmVersionString);
 		return *version;
 	}
 	else
@@ -279,7 +273,7 @@ yul::EVMDialect const& CommonOptions::evmDialect() const
 CommonOptions const& CommonOptions::get()
 {
 	if (!m_singleton)
-		BOOST_THROW_EXCEPTION(std::runtime_error("Options not yet constructed!"));
+		soltestAssert(false, "Options not yet constructed!");
 
 	return *m_singleton;
 }
@@ -306,10 +300,21 @@ bool isValidSemanticTestPath(boost::filesystem::path const& _testPath)
 	return true;
 }
 
+std::set<std::string> testFileExtensions()
+{
+	return {
+		".sol",
+		".yul",
+		".asm",
+		".asmjson", // Not .json because JSON files that do not represent test cases exist in some test dirs.
+		".stack",
+	};
+}
+
 boost::unit_test::precondition::predicate_t nonEOF()
 {
 	return [](boost::unit_test::test_unit_id) {
-		return !solidity::test::CommonOptions::get().eofVersion().has_value();
+		return !CommonOptions::get().eofVersion().has_value();
 	};
 }
 
@@ -325,13 +330,13 @@ bool loadVMs(CommonOptions const& _options)
 	if (_options.disableSemanticTests)
 		return true;
 
-	bool evmSupported = solidity::test::EVMHost::checkVmPaths(_options.vmPaths);
+	bool evmSupported = EVMHost::checkVmPaths(_options.vmPaths);
 	if (!_options.disableSemanticTests && !evmSupported)
 	{
-		std::cerr << "Unable to find " << solidity::test::evmoneFilename;
+		std::cerr << "Unable to find " << evmoneFilename;
 		std::cerr << ". Please disable semantics tests with --no-semantic-tests or provide a path using --vm <path>." << std::endl;
 		std::cerr << "You can download it at" << std::endl;
-		std::cerr << solidity::test::evmoneDownloadLink << std::endl;
+		std::cerr << evmoneDownloadLink << std::endl;
 		return false;
 	}
 	return true;

test/Common.h

@@ -48,7 +48,10 @@ static constexpr auto evmoneFilename = "libevmone.so";
 static constexpr auto evmoneDownloadLink = "https://github.com/ethereum/evmone/releases/download/v0.13.0/evmone-0.13.0-linux-x86_64.tar.gz";
 #endif
 
-struct ConfigException: public util::Exception {};
+struct SoltestError: public util::Exception {};
+struct ConfigException: public SoltestError {};
+struct ValidationError: public SoltestError {};
+struct ExecutionError: public SoltestError {};
 
 struct CommonOptions
 {
@@ -76,7 +79,7 @@ struct CommonOptions
 	virtual void addOptions();
 	// @returns true if the program should continue, false if it should exit immediately without
 	// reporting an error.
-	// Throws ConfigException or std::runtime_error if parsing fails.
+	// Throws ConfigException if parsing fails.
 	virtual bool parse(int argc, char const* const* argv);
 	// Throws a ConfigException on error
 	virtual void validate() const;
@@ -97,7 +100,7 @@ struct CommonOptions
 	boost::program_options::options_description options;
 
 private:
-	std::string evmVersionString;
+	std::string m_evmVersionString;
 	std::optional<uint8_t> m_eofVersion;
 	static std::unique_ptr<CommonOptions const> m_singleton;
 };
@@ -107,6 +110,9 @@ struct CommonOptions
 /// Note: @p _testPath can be relative but must include at least the `/test/libsolidity/semanticTests/` part
 bool isValidSemanticTestPath(boost::filesystem::path const& _testPath);
 
+/// Returns a list of file extensions allowed for test files.
+std::set<std::string> testFileExtensions();
+
 /// Helper that can be used to skip tests when the EVM version selected on the command line
 /// is older than @p _minEVMVersion.
 /// @return A predicate (function) that can be passed into @a boost::unit_test::precondition().

test/CommonSyntaxTest.cpp

@@ -30,9 +30,8 @@
 #include <boost/test/unit_test.hpp>
 #include <boost/throw_exception.hpp>
 
-#include <fstream>
-#include <memory>
-#include <stdexcept>
+#include <istream>
+#include <ostream>
 
 using namespace solidity;
 using namespace solidity::util;
@@ -42,15 +41,14 @@ using namespace solidity::frontend;
 using namespace solidity::frontend::test;
 using namespace solidity::test;
 using namespace boost::unit_test;
-namespace fs = boost::filesystem;
 
 namespace
 {
 
 int parseUnsignedInteger(std::string::iterator& _it, std::string::iterator _end)
 {
 	if (_it == _end || !util::isDigit(*_it))
-		BOOST_THROW_EXCEPTION(std::runtime_error("Invalid test expectation. Source location expected."));
+		solThrow(ValidationError, "Invalid test expectation. Source location expected.");
 	int result = 0;
 	while (_it != _end && util::isDigit(*_it))
 	{
@@ -258,8 +256,7 @@ std::vector<SyntaxTestError> CommonSyntaxTest::parseExpectations(std::istream& _
 
 		std::string errorTypeStr(typeBegin, it);
 		std::optional<Error::Type> errorType = Error::parseErrorType(errorTypeStr);
-		if (!errorType.has_value())
-			BOOST_THROW_EXCEPTION(std::runtime_error("Invalid error type: " + errorTypeStr));
+		solRequire(errorType.has_value(), ValidationError, "Invalid error type: " + errorTypeStr);
 
 		skipWhitespace(it, line.end());
 

test/CommonSyntaxTest.h

@@ -27,7 +27,6 @@
 #include <iosfwd>
 #include <string>
 #include <vector>
-#include <utility>
 
 namespace solidity::test
 {

test/EVMHost.cpp

@@ -29,6 +29,8 @@
 #endif
 
 #include <test/EVMHost.h>
+
+#include <test/Common.h>
 #include <test/libsolidity/util/SoltestErrors.h>
 
 #if defined(__GNUC__) && !defined(__clang__) // GCC-specific pragma
@@ -91,8 +93,7 @@ bool EVMHost::checkVmPaths(std::vector<boost::filesystem::path> const& _vmPaths)
 
 		if (vm.has_capability(EVMC_CAPABILITY_EVM1))
 		{
-			if (evmVmFound)
-				BOOST_THROW_EXCEPTION(std::runtime_error("Multiple evm1 evmc vms defined. Please only define one evm1 evmc vm."));
+			solRequire(!evmVmFound, ConfigException, "Multiple evm1 evmc vms defined. Please only define one evm1 evmc vm.");
 			evmVmFound = true;
 		}
 	}

test/ExecutionFramework.h

@@ -34,8 +34,6 @@
 #include <libsolutil/FunctionSelector.h>
 #include <libsolutil/ErrorCodes.h>
 
-#include <functional>
-
 #include <boost/rational.hpp>
 #include <boost/test/unit_test.hpp>