From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1QfBeP-0000Df-4r for garchives@archives.gentoo.org; Fri, 08 Jul 2011 14:03:25 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 692B421C1A4; Fri, 8 Jul 2011 14:03:03 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 163AF21C10D for ; Fri, 8 Jul 2011 14:03:03 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 870F32AC0EA for ; Fri, 8 Jul 2011 14:03:02 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 187FE80042 for ; Fri, 8 Jul 2011 14:03:01 +0000 (UTC) From: "Petteri Räty" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Petteri Räty" Message-ID: Subject: [gentoo-commits] proj/libbash:master commit in: / X-VCS-Repository: proj/libbash X-VCS-Committer: betelgeuse X-VCS-Committer-Name: Petteri Räty X-VCS-Revision: d3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7 Date: Fri, 8 Jul 2011 14:03:01 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 59f09fb531b35fcd7b7fb39f834c4e7d commit: d3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7 Author: Mu Qiao gentoo org> AuthorDate: Fri Jul 1 09:51:38 2011 +0000 Commit: Petteri R=C3=A4ty gentoo org> CommitDate: Tue Jul 5 01:49:27 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/libbash.git;a= =3Dcommit;h=3Dd3296b24 Merge remote branch 'betelgeuse/multithread' into multithreading Conflicts: src/builtins/source_builtin.cpp src/core/bash_ast.cpp src/core/bash_ast.h utils/instruo.cpp Makefile.am | 4 +- bashast/libbashWalker.g | 2 +- src/builtins/source_builtin.cpp | 51 +++++++++++-------- src/core/bash_ast.cpp | 39 +++++++++++++- src/core/bash_ast.h | 8 ++- utils/instruo.cpp | 108 ++++++++++++++++++---------------= ------ 6 files changed, 125 insertions(+), 87 deletions(-) diff --cc src/builtins/source_builtin.cpp index bec83ad,ee95e29..d7924ab --- a/src/builtins/source_builtin.cpp +++ b/src/builtins/source_builtin.cpp @@@ -30,39 -32,45 +31,47 @@@ =20 #include "builtins/builtin_exceptions.h" #include "cppbash_builtin.h" +#include "core/exceptions.h" #include "core/interpreter.h" -#include "core/interpreter_exception.h" #include "core/bash_ast.h" =20 - int source_builtin::exec(const std::vector& bash_args) - { - static std::unordered_map> ast= _cache; + namespace { + std::mutex parse_mutex; =20 - if(bash_args.size() =3D=3D 0) - throw libbash::illegal_argument_exception("should provide one argum= ent for source builtin"); + std::shared_ptr& parse(const std::string& path) + { + static std::unordered_map> a= st_cache; =20 - // we need fix this to pass extra arguments as positional parameters - const std::string& path =3D bash_args[0]; + std::lock_guard parse_lock(parse_mutex); =20 - auto stored_ast =3D ast_cache.find(path); - if(stored_ast =3D=3D ast_cache.end()) - { - // ensure the path is cached - auto iter =3D ast_cache.insert(make_pair(path, std::shared_ptr())); - // this may throw exception - iter.first->second.reset(new bash_ast(path)); - stored_ast =3D iter.first; - } - else if(!(stored_ast->second)) - { - throw libbash::parse_exception(path + " cannot be fully parsed"); - auto& stored_ast =3D ast_cache[path]; - if(!stored_ast) ++ auto stored_ast =3D ast_cache.find(path); ++ if(stored_ast =3D=3D ast_cache.end()) + { - std::ifstream input(path); - if(!input) - throw interpreter_exception(path + " can't be read"); - - stored_ast.reset(new bash_ast(input)); - if(stored_ast->get_error_count()) - std::cerr << path << " could not be parsed properly" << std::en= dl; ++ // ensure the path is cached ++ auto iter =3D ast_cache.insert(make_pair(path, std::shared_ptr())); ++ // this may throw exception ++ iter.first->second.reset(new bash_ast(path)); ++ stored_ast =3D iter.first; ++ } ++ else if(!(stored_ast->second)) ++ { ++ throw libbash::parse_exception(path + " cannot be fully parsed"); + } +=20 - return stored_ast; ++ return stored_ast->second; } + } +=20 + int source_builtin::exec(const std::vector& bash_args) + { - static std::unordered_map> ast= _cache; - + if(bash_args.size() =3D=3D 0) - throw interpreter_exception("should provide one argument for source= builtin"); ++ throw libbash::illegal_argument_exception("should provide one argum= ent for source builtin"); =20 + const std::string& original_path =3D _walker.resolve("0"= ); ++ _walker.define("0", bash_args.front(), true); try { - _walker.define("0", path, true); - stored_ast->second->interpret_with(_walker); + parse(bash_args.front())->interpret_with(_walker); } catch(return_exception& e) {} =20 diff --cc src/core/bash_ast.cpp index c7b9180,22c7f22..3787f84 --- a/src/core/bash_ast.cpp +++ b/src/core/bash_ast.cpp @@@ -18,15 -18,16 +18,17 @@@ */ /// /// \file bash_ast.cpp -/// \author Mu Qiao -/// \brief implementation that helps interpret from istream +/// \brief a wrapper class that helps interpret from istream and string /// - #include "core/bash_ast.h" =20 +#include + #include + #include =20 -#include "core/interpreter_exception.h" +#include + +#include "core/exceptions.h" #include "core/interpreter.h" #include "libbashLexer.h" #include "libbashParser.h" @@@ -38,60 -38,107 +40,85 @@@ bash_ast::bash_ast(const std::istream&=20 std::stringstream stream; stream << source.rdbuf(); script =3D stream.str(); + init_parser(script, "unknown source"); +} =20 - input =3D antlr3NewAsciiStringInPlaceStream( - reinterpret_cast(const_cast(script.c_str())= ), - script.size(), - NULL); - - if(input =3D=3D NULL) - throw interpreter_exception("Unable to open file " + script + " due= to malloc() failure"); +bash_ast::bash_ast(const std::string& script_path, + std::function p):= parse(p) +{ + std::stringstream stream; + std::ifstream file_stream(script_path); + if(!file_stream) + throw libbash::parse_exception(script_path + " can't be read"); =20 - init_parser(); + stream << file_stream.rdbuf(); + script =3D stream.str(); + init_parser(script, script_path); } =20 + namespace + { + std::mutex string_mutex; +=20 + pANTLR3_STRING locked_newRaw8(pANTLR3_STRING_FACTORY factory) + { + std::lock_guard l(string_mutex); + pANTLR3_STRING_FACTORY pristine =3D antlr3StringFactoryNew(); + pANTLR3_STRING result =3D pristine->newRaw(factory); + pristine->close(pristine); + return result; + } +=20 + void locked_destroy(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING st= ring) + { + std::lock_guard l(string_mutex); + pANTLR3_STRING_FACTORY pristine =3D antlr3StringFactoryNew(); + pristine->destroy(factory, string); + pristine->close(pristine); + } + } +=20 -bash_ast::~bash_ast() +void bash_ast::init_parser(const std::string& script, const std::string= & script_path) { - psr->free(psr); - tstream->free(tstream); - lxr->free(lxr); - input->close(input); -} - -void bash_ast::init_parser() -{ - lxr =3D libbashLexerNew(input); - if ( lxr =3D=3D NULL ) - { - std::cerr << "Unable to create the lexer due to malloc() failure" <= < std::endl; - error_count =3D 1; - return; - } - - tstream =3D antlr3CommonTokenStreamSourceNew( - ANTLR3_SIZE_HINT, lxr->pLexer->rec->state->tokSource); - if (tstream =3D=3D NULL) - { - std::cerr << "Out of memory trying to allocate token stream" << std= ::endl; - error_count =3D 1; - return; - } - - psr =3D libbashParserNew(tstream); - if (psr =3D=3D NULL) - { - std::cerr << "Out of memory trying to allocate parser" << std::endl= ; - error_count =3D 1; - return; - } - - langAST.reset(new libbashParser_start_return(psr->start(psr))); - langAST->tree->strFactory->newRaw =3D &locked_newRaw8; - langAST->tree->strFactory->destroy =3D &locked_destroy; - error_count =3D psr->pParser->rec->getNumberOfSyntaxErrors(psr->pPars= er->rec); -} - -void bash_ast::interpret_with(interpreter& walker) -{ - set_interpreter(&walker); - - auto nodes =3D antlr3CommonTreeNodeStreamNewTree(langAST->tree, ANTLR= 3_SIZE_HINT); - plibbashWalker treePsr =3D libbashWalkerNew(nodes); - try - { - treePsr->start(treePsr); - } - catch(...) - { - treePsr->free(treePsr); - throw; - } - treePsr->free(treePsr); - nodes->free(nodes); + input.reset(antlr3NewAsciiStringInPlaceStream( + reinterpret_cast(const_cast(script.c_str())), + // We do not support strings longer than the max value of ANTLR3_UN= IT32 + boost::numeric_cast(script.size()), + NULL)); + + if(!input) + throw libbash::parse_exception("Unable to open file " + script + " = due to malloc() failure"); + + input->fileName =3D input->strFactory->newStr( + input->strFactory, + reinterpret_cast(const_cast(script_path.c_s= tr()))); + + lexer.reset(libbashLexerNew(input.get())); + if(!lexer) + throw libbash::parse_exception("Unable to create the lexer due to m= alloc() failure"); + + token_stream.reset(antlr3CommonTokenStreamSourceNew( + ANTLR3_SIZE_HINT, lexer->pLexer->rec->state->tokSource)); + if(!token_stream) + throw libbash::parse_exception("Out of memory trying to allocate to= ken stream"); + + parser.reset(libbashParserNew(token_stream.get())); + if(!parser) + throw libbash::parse_exception("Out of memory trying to allocate pa= rser"); + + ast =3D parse(parser.get()); ++ ast->strFactory->newRaw =3D &locked_newRaw8; ++ ast->strFactory->destroy =3D &locked_destroy; + if(parser->pParser->rec->getNumberOfSyntaxErrors(parser->pParser->rec= )) + throw libbash::parse_exception("Something wrong happened while pars= ing"); - nodes.reset(antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT))= ; } =20 std::string bash_ast::get_dot_graph() { - auto nodes =3D antlr3CommonTreeNodeStreamNewTree(langAST->tree, ANTLR= 3_SIZE_HINT); - pANTLR3_STRING graph =3D nodes->adaptor->makeDot(nodes->adaptor, lang= AST->tree); - std::string result(reinterpret_cast(graph->chars)); - nodes->free(nodes); - return result; ++ antlr_pointer nodes( ++ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); + pANTLR3_STRING graph =3D nodes->adaptor->makeDot(nodes->adaptor, ast)= ; + return std::string(reinterpret_cast(graph->chars)); } =20 std::string bash_ast::get_string_tree() @@@ -148,71 -195,6 +175,79 @@@ std::string bash_ast::get_parser_tokens print_line_counter(result, token, line_counter, tokenName =3D=3D = "CONTINUE_LINE"? 1 : 0); } } + return result.str(); +} + +std::string bash_ast::get_walker_tokens(std::function token_map) +{ + std::stringstream result; ++ antlr_pointer nodes( ++ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); + pANTLR3_INT_STREAM istream =3D nodes->tnstream->istream; + auto istream_size =3D istream->size(istream); + + for(ANTLR3_UINT32 i =3D 1; i <=3D istream_size; ++i) + { + ANTLR3_UINT32 token =3D istream->_LA(istream, boost::numeric_cast(i)); + if(token =3D=3D 2) + result << "DOWN "; + else if(token =3D=3D 3) + result << "UP "; + else + result << token_map(istream->_LA(istream, boost::numeric_cast(i))) << " "; + } + result << std::endl; =20 return result.str(); } + +void bash_ast::walker_start(plibbashWalker tree_parser) +{ + tree_parser->start(tree_parser); +} + +long bash_ast::walker_arithmetics(plibbashWalker tree_parser) +{ + return tree_parser->arithmetics(tree_parser); +} + +pANTLR3_BASE_TREE bash_ast::parser_start(plibbashParser parser) +{ + return parser->start(parser).tree; +} + +pANTLR3_BASE_TREE bash_ast::parser_arithmetics(plibbashParser parser) +{ + return parser->arithmetics(parser).tree; +} + +void bash_ast::call_function(plibbashWalker ctx, + ANTLR3_MARKER index) +{ + auto INPUT =3D ctx->pTreeParser->ctnstream; ++ ++ // Initialize the input stream ++ auto ISTREAM =3D ctx->pTreeParser->ctnstream->tnstream->istream; ++ ISTREAM->size(ISTREAM); ++ + // Push function index into INPUT + // The actual type of ANTLR3_MARKER is ANTLR3_INT32 + INPUT->push(INPUT, boost::numeric_cast(index)); + // Execute function body + ctx->compound_command(ctx); +} + - bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker) ++bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker, ++ antlr_pointer& nodes) +{ + set_interpreter(&walker); + walker.push_current_ast(this); + + auto deleter =3D [&](plibbashWalker tree_parser) + { + tree_parser->free(tree_parser); + walker.pop_current_ast(); + }; + + return walker_pointer(libbashWalkerNew(nodes.get()), deleter); +} diff --cc src/core/bash_ast.h index 22635a1,852899c..190b020 --- a/src/core/bash_ast.h +++ b/src/core/bash_ast.h @@@ -36,79 -34,37 +36,81 @@@ =20 struct libbashLexer_Ctx_struct; struct libbashParser_Ctx_struct; -struct libbashParser_start_return_struct; +struct libbashWalker_Ctx_struct; +typedef libbashWalker_Ctx_struct* plibbashWalker; class interpreter; =20 +template +class antlr_pointer: public std::unique_ptr> +{ + typedef std::unique_ptr> parent; +public: + antlr_pointer(T* p =3D 0) : parent(p, [](T* to_delete) { to_delete->f= ree(to_delete); }) {}; +}; + /// \class bash_ast -/// \brief a wrapper class that helps interpret from istream -class bash_ast +/// \brief a wrapper class that helps interpret from istream and string +class bash_ast: public boost::noncopyable { - pANTLR3_INPUT_STREAM input; + antlr_pointer input; std::string script; - libbashLexer_Ctx_struct* lxr; - pANTLR3_COMMON_TOKEN_STREAM tstream; - libbashParser_Ctx_struct* psr; - std::unique_ptr langAST; - int error_count; + antlr_pointer lexer; + antlr_pointer token_stream; + antlr_pointer parser; + pANTLR3_BASE_TREE ast; - antlr_pointer nodes; + std::function parse; + + typedef std::unique_ptr> walker_pointer; + + void init_parser(const std::string& script, const std::string& script= _path); - walker_pointer create_walker(interpreter& walker); ++ walker_pointer create_walker(interpreter& walker, ++ antlr_pointer& nodes); =20 - void init_parser(); public: - explicit bash_ast(std::istream& source); - ~bash_ast(); + bash_ast(const std::istream& source, + std::function = p=3Dparser_start); + + bash_ast(const std::string& script_path, + std::function = p=3Dparser_start); + + static void walker_start(plibbashWalker tree_parser); + + static long walker_arithmetics(plibbashWalker tree_parser); + + static void call_function(plibbashWalker tree_parser, + ANTLR3_MARKER index); + + static pANTLR3_BASE_TREE parser_start(libbashParser_Ctx_struct* parse= r); + + static pANTLR3_BASE_TREE parser_arithmetics(libbashParser_Ctx_struct*= parser); =20 /// /// \brief interpret the script with a given interpreter /// \param the interpreter object - void interpret_with(interpreter& walker); + /// \return the interpreted result + template + typename std::result_of::type + interpret_with(interpreter& walker, Functor walk) + { - walker_pointer p_tree_parser =3D create_walker(walker); ++ antlr_pointer nodes( ++ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); ++ walker_pointer p_tree_parser =3D create_walker(walker, nodes); + return walk(p_tree_parser.get()); + } + + void interpret_with(interpreter& walker) + { + interpret_with(walker, walker_start); + } + std::string get_dot_graph(); + std::string get_string_tree(); - std::string get_tokens(std::function token= _map); + + static std::string get_parser_tokens(antlr_pointer& token_stream, + std::function); + + std::string get_walker_tokens(std::function); }; =20 #endif diff --cc utils/instruo.cpp index dcd3924,f91bbf1..704c98c --- a/utils/instruo.cpp +++ b/utils/instruo.cpp @@@ -53,7 -53,6 +53,7 @@@ #include #include =20 - #include "builtins/builtin_exceptions.h" ++#include "core/exceptions.h" #include "command_line.h" #include "libbash.h" #include "utils/metadata.h" @@@ -65,71 -64,63 +65,63 @@@ using std::endl =20 void worker(const std::shared_ptr &ids) { - std::unordered_map> variables; -=20 - std::shared_ptr id; unsigned total(0); CategoryNamePart old_cat("OLDCAT"); - while(!ids->empty()) + #pragma omp parallel { - id =3D *ids->begin(); - ids->pop_front(); - if (id->name().category() !=3D old_cat) + #pragma omp single nowait + while(!ids->empty()) { - std::cout << "Processing " << stringify(id->name().category()) <<= "..." << std::endl; - old_cat =3D id->name().category(); - FSPath(CommandLine::get_instance()->a_output_directory.argument()= + "/" + - stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_if= _exists}); - ++total; - } + std::shared_ptr id =3D *ids->begin(); + ids->pop_front(); + if (id->name().category() !=3D old_cat) + { + std::cout << "Processing " << stringify(id->name().category()) = << "..." << std::endl; + old_cat =3D id->name().category(); + FSPath(CommandLine::get_instance()->a_output_directory.argument= () + "/" + + stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_= if_exists}); + ++total; + } =20 - Context i_context("When generating metadata for ID '" + stringify(*= id) + "':"); + #pragma omp task + { + Context i_context("When generating metadata for ID '" + stringi= fy(*id) + "':"); =20 - variables.clear(); - variables["PN"].push_back(stringify(id->name().package())); - variables["PV"].push_back(stringify(id->version().remove_revision()= )); - variables["P"].push_back(stringify(id->name().package()) + "-" + - stringify(id->version().remove_revision())= ); - variables["PR"].push_back(id->version().revision_only()); - variables["PVR"].push_back(stringify(id->version())); - variables["PF"].push_back(stringify(id->name().package()) + "-" + s= tringify(id->version())); - variables["CATEGORY"].push_back(stringify(id->name().category())); - std::vector functions; + std::unordered_map> varia= bles; + variables["PN"].push_back(stringify(id->name().package())); + variables["PV"].push_back(stringify(id->version().remove_revisi= on())); + variables["P"].push_back(stringify(id->name().package()) + "-" = + + stringify(id->version().remove_revisio= n())); + variables["PR"].push_back(id->version().revision_only()); + variables["PVR"].push_back(stringify(id->version())); + variables["PF"].push_back(stringify(id->name().package()) + "-"= + stringify(id->version())); + variables["CATEGORY"].push_back(stringify(id->name().category()= )); + std::vector functions; =20 - std::string ebuild_path(CommandLine::get_instance()->a_repository_d= irectory.argument() + - variables["CATEGORY"][0] + "/" + - variables["PN"][0] + "/" + - variables["PN"][0] + "-" + - variables["PVR"][0] + ".ebuild"); - try - { - libbash::interpret(ebuild_path, "utils/isolated-functions.sh", va= riables, functions); - } - catch(const libbash::interpreter_exception& e) - { - cerr << "Exception occurred while interpreting " << ebuild_path <= < ". The error message is:\n" - << e.what() << endl; - continue; - } - catch(const return_exception& e) - { - cerr << "Unhandled return exception in " << ebuild_path << ". The= error message is:\n" - << e.what() << endl; - continue; - } - catch (...) - { - cerr << "Unhandled exception in " << ebuild_path << endl; - continue; - } + std::string ebuild_path(CommandLine::get_instance()->a_reposito= ry_directory.argument() + + variables["CATEGORY"][0] + "/" + + variables["PN"][0] + "/" + + variables["PN"][0] + "-" + + variables["PVR"][0] + ".ebuild"); + try + { + libbash::interpret(ebuild_path, variables, functions); =20 - std::string output_path(CommandLine::get_instance()->a_output_direc= tory.argument() + "/" + - variables["CATEGORY"][0] + "/" + - variables["PN"][0] + "-" + - variables["PVR"][0]); - FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exists}); - std::ofstream output(output_path, std::ofstream::out | std::ofstrea= m::trunc); - write_metadata(output, variables, functions); + std::string output_path(CommandLine::get_instance()->a_output= _directory.argument() + "/" + + variables["CATEGORY"][0] + "/" + + variables["PN"][0] + "-" + + variables["PVR"][0]); + FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exis= ts}); + std::ofstream output(output_path, std::ofstream::out | std::o= fstream::trunc); + write_metadata(output, variables, functions); + } - catch(const interpreter_exception& e) ++ catch(const libbash::interpreter_exception& e) + { + cerr << "Exception occurred while interpreting " << ebuild_pa= th << ". The error message is:\n" + << e.what() << endl; + } + } + } } } =20