Need help using member functions as action with Boost Spirit QI

Viewed 46

I'm having trouble getting member functions to bind inside grammar definitions. Compile errors result.

In short:

struct my_functor_word
{
  // This code
  void print ( std::string const& s, qi::unused_type, qi::unused_type ) const
  // Gives the compiler error seen below. 
  // This code works fine:
  // void operator()( std::string const& s, qi::unused_type, qi::unused_type ) const
  {
    std::cout << "word:" << s << std::endl;
  }
};

template <typename Iterator>
struct bd_parse_grammar : qi::grammar<Iterator>
{
  template <typename TokenDef> bd_parse_grammar( TokenDef const& tok )
    : bd_parse_grammar::base_type( start )
  {
    my_functor_word mfw;

    start =  *(
      // This code
      tok.word          [boost::bind(&my_functor_word::print, &mfw, qi::_1)]
      // gives:
      // {aka void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const}' is not a class, struct, or union type
      //       function_apply;
      //       ^~~~~~~~~~~~~~
      ///usr/include/boost/spirit/home/phoenix/core/detail/function_eval.hpp:126:13: error: 'boost::remove_reference<void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const>::type {aka void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const}' is not a class, struct, or union type
      //             type;
      //             ^~~~

      // This:
      // tok.word          [boost::bind(&my_functor_word::print, &mfw, qi::_1)]

      // similarly gives:
      // /usr/include/boost/bind/bind.hpp:69:37: error: 'void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const' is not a class, struct, or union type
      // typedef typename F::result_type type;
      
      // This works OK:
      // tok.word          [my_functor_word()] 
      ) ;
  }
  qi::rule<Iterator> start;
};

Here's the whole program. It compiles and functions correctly with functors but not member functions:

#include <boost/config/warning_disable.hpp>

#include <boost/spirit/include/qi.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_statement.hpp>
#include <boost/spirit/include/phoenix_container.hpp>
#include <boost/bind.hpp>

#include <iostream>
#include <string>

using namespace boost::spirit;
using namespace boost::spirit::ascii;

template <typename Lexer>
struct bd_parse_tokens : lex::lexer<Lexer>
{
  bd_parse_tokens()
  {
    // define patterns (lexer macros) to be used during token definition
    this->self.add_pattern( "WORD", "[a-zA-Z._]+" );

    // define tokens and associate them with the lexer
    word = "{WORD}";    // reference the pattern 'WORD' as defined above

    this->self.add ( word );
  }

  // the token 'word' exposes the matched string as its parser attribute
  lex::token_def<std::string> word;
};

///////////////////////////////////////////////////////////////////////////////
//  Grammar definition
///////////////////////////////////////////////////////////////////////////////

struct my_functor_word
{
  // This code
  void print ( std::string const& s, qi::unused_type, qi::unused_type ) const
  // Gives the compiler error seen below. 
  // This code works fine:
  // void operator()( std::string const& s, qi::unused_type, qi::unused_type ) const
  {
    std::cout << "word:" << s << std::endl;
  }
};

template <typename Iterator>
struct bd_parse_grammar : qi::grammar<Iterator>
{
  template <typename TokenDef> bd_parse_grammar( TokenDef const& tok )
    : bd_parse_grammar::base_type( start )
  {
    my_functor_word mfw;

    start =  *(
      // This code
      tok.word          [boost::bind(&my_functor_word::print, &mfw, qi::_1)]
      // gives:
      // {aka void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const}' is not a class, struct, or union type
      //       function_apply;
      //       ^~~~~~~~~~~~~~
      ///usr/include/boost/spirit/home/phoenix/core/detail/function_eval.hpp:126:13: error: 'boost::remove_reference<void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const>::type {aka void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const}' is not a class, struct, or union type
      //             type;
      //             ^~~~

      // This:
      // tok.word          [boost::bind(&my_functor_word::print, &mfw, qi::_1)]

      // similarly gives:
      // /usr/include/boost/bind/bind.hpp:69:37: error: 'void (my_functor_word::*)(const std::basic_string<char>&, boost::spirit::unused_type, boost::spirit::unused_type) const' is not a class, struct, or union type
      // typedef typename F::result_type type;
      
      // This works OK:
      // tok.word          [my_functor_word()] 
      ) ;
  }
  qi::rule<Iterator> start;
};
///////////////////////////////////////////////////////////////////////////////

int main( int argc, char* argv[] )
{
  //  Define the token type to be used: `std::string` is available as the
  //   type of the token attribute
  typedef lex::lexertl::token < char const*, boost::mpl::vector<std::string> > token_type;

  //  Define the lexer type to use implementing the state machine
  typedef lex::lexertl::lexer<token_type> lexer_type;

  //  Define the iterator type exposed by the lexer type */
  typedef bd_parse_tokens<lexer_type>::iterator_type iterator_type;

  // now we use the types defined above to create the lexer and grammar
  // object instances needed to invoke the parsing process
  bd_parse_tokens<lexer_type> bd_parse;          // Our lexer
  bd_parse_grammar<iterator_type> g( bd_parse ); // Our parser

  // read in the file int memory
  std::string str( argv[1] );
  char const* first = str.c_str();
  char const* last = &first[str.size()];

  bool r = lex::tokenize_and_parse( first, last, bd_parse, g );

  if ( ! r )
  {
    std::string rest( first, last );
    std::cerr << "Parsing failed\n" << "stopped at: \""
          << rest << "\"\n";
  }

  return 0;
}
1 Answers

There are many aspects to this puzzle.

Firstly, to bind a member function you have to pass the extra leading instance parameter (the this object).

Secondly, semantic actions are Phoenix Actors, so deferred functors. boost::bind are likely not what you wantL you cannot call my_functor_word::print with qi::_1_type anyways. Instead you might use phoenix::bind, in which case you don't even have to deal with the "magic" context parameters:

struct my_functor_word {
    void print(std::string const& s) const {
        std::cout << "word:" << s << std::endl;
    }
};

And

start = *(tok.word[ //
    boost::phoenix::bind(&my_functor_word::print, &mfw, qi::_1)]);

That Was It?

I won't let you go without some more observations.

Firstly, the bind is still broken! You bound mfw as the this instance, but mfw is a local variable. The nature of the semantic action (being a defferred actor as said above) is that it will be called during parse: long after the constructor is finished. The mfw needs to be a member variable. Or better, not be a part of the grammar at all. I'd suggest

///////////////////////////////////////////////////////////////////////////////
//  Grammar definition
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator>
struct bd_parse_grammar : qi::grammar<Iterator>
{
    template <typename TokenDef>
    bd_parse_grammar(TokenDef const& tok) : bd_parse_grammar::base_type(start)
    {
        using namespace qi::labels;
        start = *tok.word[px::bind(&my_functor_word::print, &mfw, _1)];
    }

  private:
    struct my_functor_word {
        void print(std::string const& s)
        {
            std::cout << "word:" << s << std::endl;
        }
    };

    mutable my_functor_word mfw;
    qi::rule<Iterator>      start;
};
///////////////////////////////////////////////////////////////////////////////

I see a lot of old-fashioned and questionable style. E.g. why are you including boost/bind.hpp (or even boost/bind/bind.hpp) and even boost/lambda.hpp?

Why are you using Lex?

You're using using namespace liberally, which is a bad idea, especially when mixing all these libraries (that literally all have their own idea of placeholders named 1 etc). Instead just make some aliases:

namespace qi  = boost::spirit::qi;
namespace lex = boost::spirit::lex;
namespace px  = boost::phoenix;

You have a comment

// read in the file int memory

That doesn't match the code given:

std::string str(argv[1]);
char const* first = str.c_str();
char const* last  = &first[str.size()];

That is just weird all around. Why are you using raw char* with a string (it has proper iterators with begin() and end()?). Also, maybe you really wanted to read a file?

By the way, let's make sure argv[1] is actually valid:

for (std::string fname : std::vector(argv + 1, argv + argc)) {
    std::ifstream ifs(fname);

    // read in the file into memory
    std::string const str(std::istreambuf_iterator<char>(ifs), {});

    auto first = str.begin(), //
        last   = str.end();

So here's a demo Live On Coliru

#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <fstream>
#include <iomanip>

namespace qi  = boost::spirit::qi;
namespace lex = boost::spirit::lex;
namespace px  = boost::phoenix;

template <typename Lexer> struct bd_parse_tokens : lex::lexer<Lexer> {
    bd_parse_tokens() {
        this->self.add_pattern("WORD", "[a-zA-Z._]+");
        word = "{WORD}";
        this->self.add(word);
    }

    // the token 'word' exposes the matched string as its parser attribute
    lex::token_def<std::string> word;
};

///////////////////////////////////////////////////////////////////////////////
//  Grammar definition
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator>
struct bd_parse_grammar : qi::grammar<Iterator>
{
    template <typename TokenDef>
    bd_parse_grammar(TokenDef const& tok) : bd_parse_grammar::base_type(start) {
        using namespace qi::labels;
        start = *tok.word[px::bind(&my_functor_word::print, &mfw, _1)];
    }

  private:
    struct my_functor_word {
        void print(std::string const& s) const { std::cout << "word:" << s << std::endl; }
    };

    mutable my_functor_word mfw;
    qi::rule<Iterator>      start;
};
///////////////////////////////////////////////////////////////////////////////

int main(int argc, char* argv[])
{
    // type of the token attribute
    using token_type    = lex::lexertl::token<std::string::const_iterator,
                                           boost::mpl::vector<std::string>>;
    using lexer_type    = lex::lexertl::/*actor_*/lexer<token_type>;
    using iterator_type = bd_parse_tokens<lexer_type>::iterator_type;

    bd_parse_tokens<lexer_type>     bd_parse;
    bd_parse_grammar<iterator_type> g(bd_parse);

    for (std::string fname : std::vector(argv + 1, argv + argc)) {
        std::ifstream ifs(fname);

        // read in the file into memory
        std::string const str(std::istreambuf_iterator<char>(ifs), {});

        auto first = str.begin();
        auto last  = str.end();

        bool ok = lex::tokenize_and_parse(first, last, bd_parse, g);

        std::cerr << "Parsing " << fname << " (length " << str.length() << ") "
                  << (ok ? "succeeded" : "failed") << "\n";

        if (first != last)
            std::cerr << "Stopped at #" << std::distance(str.begin(), first)
                      << "\n";
    }
}

Prints

word:includeboostspiritincludelex_lexertl.hppincludeboostspiritincludephoenix.hppincludeboostspiritincludeqi.hppincludefstreamincludeiomanipnamespaceqiboostspiritqinamespacelexboostspiritlexnamespacepxboostphoenixtemplatetypenameLexerstructbd_parse_tokenslexlexerLexerbd_parse_tokensthisself.add_patternWORDazAZ._wordWORDthisself.addwordthetokenwordexposesthematchedstringasitsparserattributelextoken_defstdstringwordGrammardefinitiontemplatetypenameIteratorstructbd_parse_grammarqigrammarIteratortemplatetypenameTokenDefbd_parse_grammarTokenDefconsttokbd_parse_grammarbase_typestartusingnamespaceqilabelsstarttok.wordpxbindmy_functor_wordprintmfw_privatestructmy_functor_wordvoidprintstdstringconstsconststdcoutwordsstdendlmutablemy_functor_wordmfwqiruleIteratorstartintmainintargccharargvtypeofthetokenattributeusingtoken_typelexlexertltokenstdstringconst_iteratorboostmplvectorstdstringusinglexer_typelexlexertlactor_lexertoken_typeusingiterator_typebd_parse_tokenslexer_typeiterator_typebd_parse_tokenslexer_typebd_parsebd_parse_grammariterator_typegbd_parseforstdstringfnamestdvectorargvargvargcstdifstreamifsfnamereadinthefileintomemorystdstringconststrstdistreambuf_iteratorcharifsautofirststr.beginautolaststr.endbooloklextokenize_and_parsefirstlastbd_parsegstdcerrParsingfnamelengthstr.lengthoksucceededfailedniffirstlaststdcerrStoppedatstddistancestr.beginfirstn
Parsing input.txt (length 1367) succeeded
Parsing main.cpp (length 2479) succeeded
Stopped at #0

Without Lex

I think this would be strictly simpler:

Live On Coliru

#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <fstream>
#include <iomanip>

namespace qi  = boost::spirit::qi;
namespace px  = boost::phoenix;

template <typename Iterator>
struct bd_parse_grammar : qi::grammar<Iterator> {
    bd_parse_grammar() : bd_parse_grammar::base_type(start)
    {
        using namespace qi::labels;
        word  = +qi::char_("a-zA-Z_.");
        start = *word[px::bind(&my_functor_word::print, &mfw, _1)];
    }

  private:
    struct my_functor_word {
        void print(std::string const& s) const { std::cout << "word:" << s << std::endl; }
    };

    mutable my_functor_word mfw;
    qi::rule<Iterator>                start;
    qi::rule<Iterator, std::string()> word;
};

int main(int argc, char* argv[]) {
    bd_parse_grammar<std::string::const_iterator> const g;

    for (std::string fname : std::vector(argv + 1, argv + argc)) {
        std::ifstream ifs(fname);

        // read the file into memory
        std::string const str(std::istreambuf_iterator<char>(ifs), {});

        auto first = str.begin();
        auto last  = str.end();

        bool ok = qi::parse(first, last, g);

        std::cerr << "Parsing " << fname << " (length " << str.length() << ") "
                  << (ok ? "succeeded" : "failed") << "\n";

        if (first != last)
            std::cerr << "Stopped at #" << std::distance(str.begin(), first)
                      << "\n";
    }
}

Same output

Without Phoenix Bind

Using some C++17 CTAD and phoenix::function:

Live On Coliru

template <typename Iterator> struct Parser : qi::grammar<Iterator> {
    Parser() : Parser::base_type(start) {
        px::function print{[](std::string const& s) {
            std::cout << "word:" << s << std::endl;
        }};

        word  = +qi::char_("a-zA-Z_.");
        start = *word[print(qi::_1)];
    }
    qi::rule<Iterator>                start;
    qi::rule<Iterator, std::string()> word;
};

Only half the original code.

Using X3

If you're using C++14 anyways, consider slashing compile times:

Live On Coliru

#include <boost/spirit/home/x3.hpp>
#include <iostream>
#include <fstream>
namespace x3 = boost::spirit::x3;

namespace Parser {
    auto print = [](auto& ctx) {
        std::cout << "word:" << _attr(ctx) << std::endl;
    };
    auto word  = +x3::char_("a-zA-Z_.");
    auto start = *word[print];
} // namespace Parser

int main(int argc, char* argv[]) {
    for (std::string fname : std::vector(argv + 1, argv + argc)) {
        std::ifstream ifs(fname);

        // read the file into memory
        std::string const str(std::istreambuf_iterator<char>(ifs), {});

        auto first = str.begin();
        auto last  = str.end();

        bool ok = x3::parse(first, last, Parser::start);

        std::cerr << "Parsing " << fname << " (length " << str.length() << ") "
                  << (ok ? "succeeded" : "failed") << "\n";

        if (first != last)
            std::cerr << "Stopped at #" << std::distance(str.begin(), first) << "\n";
    }
}

Still the same output.

Related