I have been trying to port invRegex.py to a node.js implementation for a while, but I'm still struggling with it. I already have the regular expression parse tree thanks to the ret.js tokenizer and it works pretty well, but the actual generation and concatenation of all the distinct elements in a way that is memory-efficient is revealing very challenging to me. To keep it simple, lets say I have the following regex:
[01]{1,2}@[a-f]
Feeding that to invRegex.py produces the following output (tabbified to take less space):
0@a 0@b 0@c 0@d 0@e 0@f
00@a 00@b 00@c 00@d 00@e 00@f
01@a 01@b 01@c 01@d 01@e 01@f
1@a 1@b 1@c 1@d 1@e 1@f
10@a 10@b 10@c 10@d 10@e 10@f
11@a 11@b 11@c 11@d 11@e 11@f
Considering I'm able to get each individual token and produce an array of all the valid individual outputs:
[01]{1,2} = function () {
return ['0', '00', '01', '1', '10', '11'];
};
@ = function () {
return ['@'];
};
[a-f] = function () {
return ['a', 'b', 'c', 'd', 'e', 'f'];
};
I can compute the cartesian product of all the arrays and get the same expected output:
var _ = require('underscore');
function cartesianProductOf() {
return _.reduce(arguments, function(a, b) {
return _.flatten(_.map(a, function(x) {
return _.map(b, function(y) {
return x.concat([y]);
});
}), true);
}, [ [] ]);
};
var tokens = [
['0', '00', '01', '1', '10', '11'],
['@'],
['a', 'b', 'c', 'd', 'e', 'f'],
];
var result = cartesianProductOf(tokens[0], tokens[1], tokens[2]);
_.each(result, function (value, key) {
console.log(value.join(''));
});
The problem with this is that it holds all the 36 values in memory, if I had a slightly more complicated regular expression, such as [a-z]{0,10} it would hold 146813779479511 values in memory, which is totally unfeasible. I would like to process this huge list in an asynchronous fashion, passing each generated combination to a callback and allowing me to interrupt the process at any sensible point I see fit, much like invRegex.py or this Haskell package - unfortunately I can't understand Haskell and I don't know how to mimic the generator behavior in Python to Javascript either.
I tried running a couple of simple generator experiments in node 0.11.9 (with --harmony) like this one:
function* alpha() {
yield 'a'; yield 'b'; yield 'c';
}
function* numeric() {
yield '0'; yield '1';
}
function* alphanumeric() {
yield* alpha() + numeric(); // what's the diff between yield and yield*?
}
for (var i of alphanumeric()) {
console.log(i);
}
Needless to say the above doesn't work. =/
Banging my head against the wall here, so any help tackling this problem would be highly appreciated.
UPDATE: Here is a sample ret.js parse tree for b[a-z]{3}:
{
"type": ret.types.ROOT,
"stack": [
{
"type": ret.types.CHAR,
"value": 98 // b
},
{
"type": ret.types.REPETITION,
"max": 3,
"min": 3,
"value": {
"type": ret.types.SET,
"not": false,
"set": [
{
"type": ret.types.RANGE,
"from": 97, // a
"to": 122 // z
}
]
}
}
]
]
}
The SET / RANGE type should yield 26 distinct values, and the parent REPETITION type should take that previous value to the power of 3, yielding 17576 distinct combinations. If I was to generate a flattened out tokens array like I did before for cartesianProductOf, the intermediate flattened values would take as much space as the actual cartesian product itself.
I hope this example explains better the problem I am facing.