router.d 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431
  1. /**
  2. Pattern based URL router for HTTP request.
  3. See `URLRouter` for more details.
  4. Copyright: © 2012-2015 Sönke Ludwig
  5. License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
  6. Authors: Sönke Ludwig
  7. */
  8. module vibe.http.router;
  9. public import vibe.http.server;
  10. import vibe.core.log;
  11. import std.digest.murmurhash : MurmurHash3;
  12. import std.functional;
  13. /**
  14. Routes HTTP requests based on the request method and URL.
  15. Routes are matched using a special URL match string that supports two forms
  16. of placeholders. See the sections below for more details.
  17. Registered routes are matched according to the same sequence as initially
  18. specified using `match`, `get`, `post` etc. Matching ends as soon as a route
  19. handler writes a response using `res.writeBody()` or similar means. If no
  20. route matches or if no route handler writes a response, the router will
  21. simply not handle the request and the HTTP server will automatically
  22. generate a 404 error.
  23. Match_patterns:
  24. Match patterns are character sequences that can optionally contain
  25. placeholders or raw wildcards ("*"). Raw wild cards match any character
  26. sequence, while placeholders match only sequences containing no slash
  27. ("/") characters.
  28. Placeholders are started using a colon (":") and are directly followed
  29. by their name. The first "/" character (or the end of the match string)
  30. denotes the end of the placeholder name. The part of the string that
  31. matches a placeholder will be stored in the `HTTPServerRequest.params`
  32. map using the placeholder name as the key.
  33. Match strings are subject to the following rules:
  34. $(UL
  35. $(LI A raw wildcard ("*") may only occur at the end of the match string)
  36. $(LI At least one character must be placed between any two placeholders or wildcards)
  37. $(LI The maximum allowed number of placeholders in a single match string is 64)
  38. )
  39. Match_String_Examples:
  40. $(UL
  41. $(LI `"/foo/bar"` matches only `"/foo/bar"` itself)
  42. $(LI `"/foo/*"` matches `"/foo/"`, `"/foo/bar"`, `"/foo/bar/baz"` or _any other string beginning with `"/foo/"`)
  43. $(LI `"/:x/"` matches `"/foo/"`, `"/bar/"` and similar strings (and stores `"foo"`/`"bar"` in `req.params["x"]`), but not `"/foo/bar/"`)
  44. $(LI Matching partial path entries with wildcards is possible: `"/foo:x"` matches `"/foo"`, `"/foobar"`, but not `"/foo/bar"`)
  45. $(LI Multiple placeholders and raw wildcards can be combined: `"/:x/:y/*"`)
  46. )
  47. */
  48. final class URLRouter : HTTPServerRequestHandler {
  49. @safe:
  50. private {
  51. MatchTree!Route m_routes;
  52. string m_prefix;
  53. bool m_computeBasePath;
  54. }
  55. this(string prefix = null)
  56. {
  57. m_prefix = prefix;
  58. }
  59. /** Sets a common prefix for all registered routes.
  60. All routes will implicitly have this prefix prepended before being
  61. matched against incoming requests.
  62. */
  63. @property string prefix() const { return m_prefix; }
  64. /** Controls the computation of the "routerRootDir" parameter.
  65. This parameter is available as `req.params["routerRootDir"]` and
  66. contains the relative path to the base path of the router. The base
  67. path is determined by the `prefix` property.
  68. Note that this feature currently is requires dynamic memory allocations
  69. and is opt-in for this reason.
  70. */
  71. @property void enableRootDir(bool enable) { m_computeBasePath = enable; }
  72. /// Returns a single route handle to conveniently register multiple methods.
  73. URLRoute route(string path)
  74. in { assert(path.length, "Cannot register null or empty path!"); }
  75. do { return URLRoute(this, path); }
  76. ///
  77. unittest {
  78. void getFoo(scope HTTPServerRequest req, scope HTTPServerResponse res) { /* ... */ }
  79. void postFoo(scope HTTPServerRequest req, scope HTTPServerResponse res) { /* ... */ }
  80. void deleteFoo(scope HTTPServerRequest req, scope HTTPServerResponse res) { /* ... */ }
  81. auto r = new URLRouter;
  82. // using 'with' statement
  83. with (r.route("/foo")) {
  84. get(&getFoo);
  85. post(&postFoo);
  86. delete_(&deleteFoo);
  87. }
  88. // using method chaining
  89. r.route("/foo")
  90. .get(&getFoo)
  91. .post(&postFoo)
  92. .delete_(&deleteFoo);
  93. // without using route()
  94. r.get("/foo", &getFoo);
  95. r.post("/foo", &postFoo);
  96. r.delete_("/foo", &deleteFoo);
  97. }
  98. /// Adds a new route for requests matching the specified HTTP method and pattern.
  99. URLRouter match(Handler)(HTTPMethod method, string path, Handler handler)
  100. if (isValidHandler!Handler)
  101. {
  102. import vibe.core.path : InetPath, PosixPath;
  103. import vibe.textfilter.urlencode : urlDecode;
  104. import std.algorithm : count, map;
  105. assert(path.length, "Cannot register null or empty path!");
  106. assert(count(path, ':') <= maxRouteParameters, "Too many route parameters");
  107. logDebug("add route %s %s", method, path);
  108. // Perform URL-encoding on the path before adding it as a route.
  109. string iPath = PosixPath(path)
  110. .bySegment
  111. .map!(s => InetPath.Segment(urlDecode(s.name), s.separator))
  112. .InetPath
  113. .toString;
  114. m_routes.addTerminal(iPath, Route(method, iPath, handlerDelegate(handler)));
  115. return this;
  116. }
  117. /// ditto
  118. URLRouter match(HTTPMethod method, string path, HTTPServerRequestDelegate handler)
  119. {
  120. return match!HTTPServerRequestDelegate(method, path, handler);
  121. }
  122. /// Adds a new route for GET requests matching the specified pattern.
  123. URLRouter get(Handler)(string url_match, Handler handler) if (isValidHandler!Handler) { return match(HTTPMethod.GET, url_match, handler); }
  124. /// ditto
  125. URLRouter get(string url_match, HTTPServerRequestDelegate handler) { return match(HTTPMethod.GET, url_match, handler); }
  126. /// Adds a new route for POST requests matching the specified pattern.
  127. URLRouter post(Handler)(string url_match, Handler handler) if (isValidHandler!Handler) { return match(HTTPMethod.POST, url_match, handler); }
  128. /// ditto
  129. URLRouter post(string url_match, HTTPServerRequestDelegate handler) { return match(HTTPMethod.POST, url_match, handler); }
  130. /// Adds a new route for PUT requests matching the specified pattern.
  131. URLRouter put(Handler)(string url_match, Handler handler) if (isValidHandler!Handler) { return match(HTTPMethod.PUT, url_match, handler); }
  132. /// ditto
  133. URLRouter put(string url_match, HTTPServerRequestDelegate handler) { return match(HTTPMethod.PUT, url_match, handler); }
  134. /// Adds a new route for DELETE requests matching the specified pattern.
  135. URLRouter delete_(Handler)(string url_match, Handler handler) if (isValidHandler!Handler) { return match(HTTPMethod.DELETE, url_match, handler); }
  136. /// ditto
  137. URLRouter delete_(string url_match, HTTPServerRequestDelegate handler) { return match(HTTPMethod.DELETE, url_match, handler); }
  138. /// Adds a new route for PATCH requests matching the specified pattern.
  139. URLRouter patch(Handler)(string url_match, Handler handler) if (isValidHandler!Handler) { return match(HTTPMethod.PATCH, url_match, handler); }
  140. /// ditto
  141. URLRouter patch(string url_match, HTTPServerRequestDelegate handler) { return match(HTTPMethod.PATCH, url_match, handler); }
  142. /// Adds a new route for requests matching the specified pattern, regardless of their HTTP verb.
  143. URLRouter any(Handler)(string url_match, Handler handler)
  144. {
  145. import std.traits;
  146. static HTTPMethod[] all_methods = [EnumMembers!HTTPMethod];
  147. foreach(immutable method; all_methods)
  148. match(method, url_match, handler);
  149. return this;
  150. }
  151. /// ditto
  152. URLRouter any(string url_match, HTTPServerRequestDelegate handler) { return any!HTTPServerRequestDelegate(url_match, handler); }
  153. /** Rebuilds the internal matching structures to account for newly added routes.
  154. This should be used after a lot of routes have been added to the router, to
  155. force eager computation of the match structures. The alternative is to
  156. let the router lazily compute the structures when the first request happens,
  157. which can delay this request.
  158. */
  159. void rebuild()
  160. {
  161. m_routes.rebuildGraph();
  162. }
  163. /// Handles a HTTP request by dispatching it to the registered route handlers.
  164. void handleRequest(HTTPServerRequest req, HTTPServerResponse res)
  165. {
  166. import vibe.textfilter.urlencode : urlDecode;
  167. auto method = req.method;
  168. string calcBasePath()
  169. @safe {
  170. import vibe.core.path : InetPath, relativeToWeb;
  171. auto p = InetPath(prefix.length ? prefix : "/");
  172. p.endsWithSlash = true;
  173. return p.relativeToWeb(req.requestPath).toString();
  174. }
  175. string path;
  176. // NOTE: Instead of failing, we just ignore requests with invalid path
  177. // segments (i.e. containing path separators) here. Any request
  178. // handlers later in the queue may still choose to process them
  179. // appropriately.
  180. try path = req.requestPath.toString();
  181. catch (Exception e) return;
  182. if (path.length < m_prefix.length || path[0 .. m_prefix.length] != m_prefix) return;
  183. path = path[m_prefix.length .. $];
  184. while (true) {
  185. bool done = m_routes.match(path, (ridx, scope values) @safe {
  186. auto r = () @trusted { return &m_routes.getTerminalData(ridx); } ();
  187. if (r.method != method) return false;
  188. logDebugV("route match: %s -> %s %s %s", req.requestPath, r.method, r.pattern, values);
  189. foreach (i, v; values) {
  190. req.params[m_routes.getTerminalVarNames(ridx)[i]] = urlDecode(v);
  191. }
  192. if (m_computeBasePath) req.params["routerRootDir"] = calcBasePath();
  193. r.cb(req, res);
  194. return res.headerWritten;
  195. });
  196. if (done) return;
  197. if (method == HTTPMethod.HEAD) method = HTTPMethod.GET;
  198. else break;
  199. }
  200. logDebug("no route match: %s %s", req.method, req.requestURL);
  201. }
  202. /// Returns all registered routes as const AA
  203. const(Route)[] getAllRoutes()
  204. {
  205. auto routes = new Route[m_routes.terminalCount];
  206. foreach (i, ref r; routes)
  207. r = m_routes.getTerminalData(i);
  208. return routes;
  209. }
  210. template isValidHandler(Handler) {
  211. @system {
  212. alias USDel = void delegate(HTTPServerRequest, HTTPServerResponse) @system;
  213. alias USFun = void function(HTTPServerRequest, HTTPServerResponse) @system;
  214. alias USDelS = void delegate(scope HTTPServerRequest, scope HTTPServerResponse) @system;
  215. alias USFunS = void function(scope HTTPServerRequest, scope HTTPServerResponse) @system;
  216. }
  217. static if (
  218. is(Handler : HTTPServerRequestDelegate) ||
  219. is(Handler : HTTPServerRequestFunction) ||
  220. is(Handler : HTTPServerRequestHandler) ||
  221. is(Handler : HTTPServerRequestDelegateS) ||
  222. is(Handler : HTTPServerRequestFunctionS) ||
  223. is(Handler : HTTPServerRequestHandlerS)
  224. )
  225. {
  226. enum isValidHandler = true;
  227. } else static if (
  228. is(Handler : USDel) || is(Handler : USFun) ||
  229. is(Handler : USDelS) || is(Handler : USFunS)
  230. )
  231. {
  232. enum isValidHandler = true;
  233. } else {
  234. enum isValidHandler = false;
  235. }
  236. }
  237. static void delegate(HTTPServerRequest, HTTPServerResponse) @safe handlerDelegate(Handler)(Handler handler)
  238. {
  239. import std.traits : isFunctionPointer;
  240. static if (isFunctionPointer!Handler) return handlerDelegate(() @trusted { return toDelegate(handler); } ());
  241. else static if (is(Handler == class) || is(Handler == interface)) return &handler.handleRequest;
  242. else static if (__traits(compiles, () @safe { handler(HTTPServerRequest.init, HTTPServerResponse.init); } ())) return handler;
  243. else return (req, res) @trusted { handler(req, res); };
  244. }
  245. unittest {
  246. static assert(isValidHandler!HTTPServerRequestFunction);
  247. static assert(isValidHandler!HTTPServerRequestDelegate);
  248. static assert(isValidHandler!HTTPServerRequestHandler);
  249. static assert(isValidHandler!HTTPServerRequestFunctionS);
  250. static assert(isValidHandler!HTTPServerRequestDelegateS);
  251. static assert(isValidHandler!HTTPServerRequestHandlerS);
  252. static assert(isValidHandler!(void delegate(HTTPServerRequest req, HTTPServerResponse res) @system));
  253. static assert(isValidHandler!(void function(HTTPServerRequest req, HTTPServerResponse res) @system));
  254. static assert(isValidHandler!(void delegate(scope HTTPServerRequest req, scope HTTPServerResponse res) @system));
  255. static assert(isValidHandler!(void function(scope HTTPServerRequest req, scope HTTPServerResponse res) @system));
  256. static assert(!isValidHandler!(int delegate(HTTPServerRequest req, HTTPServerResponse res) @system));
  257. static assert(!isValidHandler!(int delegate(HTTPServerRequest req, HTTPServerResponse res) @safe));
  258. void test(H)(H h)
  259. {
  260. static assert(isValidHandler!H);
  261. }
  262. test((HTTPServerRequest req, HTTPServerResponse res) {});
  263. }
  264. }
  265. ///
  266. @safe unittest {
  267. import vibe.http.fileserver;
  268. void addGroup(HTTPServerRequest req, HTTPServerResponse res)
  269. {
  270. // Route variables are accessible via the params map
  271. logInfo("Getting group %s for user %s.", req.params["groupname"], req.params["username"]);
  272. }
  273. void deleteUser(HTTPServerRequest req, HTTPServerResponse res)
  274. {
  275. // ...
  276. }
  277. void auth(HTTPServerRequest req, HTTPServerResponse res)
  278. {
  279. // TODO: check req.session to see if a user is logged in and
  280. // write an error page or throw an exception instead.
  281. }
  282. void setup()
  283. {
  284. auto router = new URLRouter;
  285. // Matches all GET requests for /users/*/groups/* and places
  286. // the place holders in req.params as 'username' and 'groupname'.
  287. router.get("/users/:username/groups/:groupname", &addGroup);
  288. // Matches all requests. This can be useful for authorization and
  289. // similar tasks. The auth method will only write a response if the
  290. // user is _not_ authorized. Otherwise, the router will fall through
  291. // and continue with the following routes.
  292. router.any("*", &auth);
  293. // Matches a POST request
  294. router.post("/users/:username/delete", &deleteUser);
  295. // Matches all GET requests in /static/ such as /static/img.png or
  296. // /static/styles/sty.css
  297. router.get("/static/*", serveStaticFiles("public/"));
  298. // Setup a HTTP server...
  299. auto settings = new HTTPServerSettings;
  300. // ...
  301. // The router can be directly passed to the listenHTTP function as
  302. // the main request handler.
  303. listenHTTP(settings, router);
  304. }
  305. }
  306. /** Using nested routers to map components to different sub paths. A component
  307. could for example be an embedded blog engine.
  308. */
  309. @safe unittest {
  310. // some embedded component:
  311. void showComponentHome(HTTPServerRequest req, HTTPServerResponse res)
  312. {
  313. // ...
  314. }
  315. void showComponentUser(HTTPServerRequest req, HTTPServerResponse res)
  316. {
  317. // ...
  318. }
  319. void registerComponent(URLRouter router)
  320. {
  321. router.get("/", &showComponentHome);
  322. router.get("/users/:user", &showComponentUser);
  323. }
  324. // main application:
  325. void showHome(HTTPServerRequest req, HTTPServerResponse res)
  326. {
  327. // ...
  328. }
  329. void setup()
  330. {
  331. auto c1router = new URLRouter("/component1");
  332. registerComponent(c1router);
  333. auto mainrouter = new URLRouter;
  334. mainrouter.get("/", &showHome);
  335. // forward all unprocessed requests to the component router
  336. mainrouter.any("*", c1router);
  337. // now the following routes will be matched:
  338. // / -> showHome
  339. // /component1/ -> showComponentHome
  340. // /component1/users/:user -> showComponentUser
  341. // Start the HTTP server
  342. auto settings = new HTTPServerSettings;
  343. // ...
  344. listenHTTP(settings, mainrouter);
  345. }
  346. }
  347. @safe unittest { // issue #1668
  348. auto r = new URLRouter;
  349. r.get("/", (req, res) {
  350. if ("foo" in req.headers)
  351. res.writeBody("bar");
  352. });
  353. r.get("/", (scope req, scope res) {
  354. if ("foo" in req.headers)
  355. res.writeBody("bar");
  356. });
  357. r.get("/", (req, res) {});
  358. r.post("/", (req, res) {});
  359. r.put("/", (req, res) {});
  360. r.delete_("/", (req, res) {});
  361. r.patch("/", (req, res) {});
  362. r.any("/", (req, res) {});
  363. }
  364. @safe unittest { // issue #1866
  365. auto r = new URLRouter;
  366. r.match(HTTPMethod.HEAD, "/foo", (scope req, scope res) { res.writeVoidBody; });
  367. r.match(HTTPMethod.HEAD, "/foo", (req, res) { res.writeVoidBody; });
  368. r.match(HTTPMethod.HEAD, "/foo", (scope HTTPServerRequest req, scope HTTPServerResponse res) { res.writeVoidBody; });
  369. r.match(HTTPMethod.HEAD, "/foo", (HTTPServerRequest req, HTTPServerResponse res) { res.writeVoidBody; });
  370. auto r2 = new URLRouter;
  371. r.match(HTTPMethod.HEAD, "/foo", r2);
  372. }
  373. @safe unittest {
  374. import vibe.inet.url;
  375. auto router = new URLRouter;
  376. string result;
  377. void a(HTTPServerRequest req, HTTPServerResponse) { result ~= "A"; }
  378. void b(HTTPServerRequest req, HTTPServerResponse) { result ~= "B"; }
  379. void c(HTTPServerRequest req, HTTPServerResponse) { assert(req.params["test"] == "x", "Wrong variable contents: "~req.params["test"]); result ~= "C"; }
  380. void d(HTTPServerRequest req, HTTPServerResponse) { assert(req.params["test"] == "y", "Wrong variable contents: "~req.params["test"]); result ~= "D"; }
  381. void e(HTTPServerRequest req, HTTPServerResponse) { assert(req.params["test"] == "z/z", "Wrong variable contents: "~req.params["test"]); result ~= "E"; }
  382. void f(HTTPServerRequest req, HTTPServerResponse) { result ~= "F"; }
  383. router.get("/test", &a);
  384. router.post("/test", &b);
  385. router.get("/a/:test", &c);
  386. router.get("/a/:test/", &d);
  387. router.get("/e/:test", &e);
  388. router.get("/f%2F", &f);
  389. auto res = createTestHTTPServerResponse();
  390. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/")), res);
  391. assert(result == "", "Matched for non-existent / path");
  392. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/test")), res);
  393. assert(result == "A", "Didn't match a simple GET request");
  394. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/test"), HTTPMethod.POST), res);
  395. assert(result == "AB", "Didn't match a simple POST request");
  396. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/a/"), HTTPMethod.GET), res);
  397. assert(result == "AB", "Matched empty variable. "~result);
  398. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/a/x"), HTTPMethod.GET), res);
  399. assert(result == "ABC", "Didn't match a trailing 1-character var.");
  400. // currently fails due to Path not accepting "//"
  401. //router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/a//"), HTTPMethod.GET), res);
  402. //assert(result == "ABC", "Matched empty string or slash as var. "~result);
  403. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/a/y/"), HTTPMethod.GET), res);
  404. assert(result == "ABCD", "Didn't match 1-character infix variable.");
  405. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/e/z%2Fz"), HTTPMethod.GET), res);
  406. assert(result == "ABCDE", "URL-escaped '/' confused router.");
  407. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/f%2F"), HTTPMethod.GET), res);
  408. assert(result == "ABCDEF", "Unable to match '%2F' in path.");
  409. }
  410. @safe unittest {
  411. import vibe.inet.url;
  412. auto router = new URLRouter("/test");
  413. string result;
  414. void a(HTTPServerRequest req, HTTPServerResponse) { result ~= "A"; }
  415. void b(HTTPServerRequest req, HTTPServerResponse) { result ~= "B"; }
  416. router.get("/x", &a);
  417. router.get("/y", &b);
  418. auto res = createTestHTTPServerResponse();
  419. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/test")), res);
  420. assert(result == "");
  421. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/test/x")), res);
  422. assert(result == "A");
  423. router.handleRequest(createTestHTTPServerRequest(URL("http://localhost/test/y")), res);
  424. assert(result == "AB");
  425. }
  426. @safe unittest {
  427. string ensureMatch(string pattern, string local_uri, string[string] expected_params = null)
  428. {
  429. import vibe.inet.url : URL;
  430. string ret = local_uri ~ " did not match " ~ pattern;
  431. auto r = new URLRouter;
  432. r.get(pattern, (req, res) {
  433. ret = null;
  434. foreach (k, v; expected_params) {
  435. if (k !in req.params) { ret = "Parameter "~k~" was not matched."; return; }
  436. if (req.params[k] != v) { ret = "Parameter "~k~" is '"~req.params[k]~"' instead of '"~v~"'."; return; }
  437. }
  438. });
  439. auto req = createTestHTTPServerRequest(URL("http://localhost"~local_uri));
  440. auto res = createTestHTTPServerResponse();
  441. r.handleRequest(req, res);
  442. return ret;
  443. }
  444. assert(ensureMatch("/foo bar/", "/foo%20bar/") is null); // normalized pattern: "/foo%20bar/"
  445. assert(ensureMatch("/foo%20bar/", "/foo%20bar/") is null); // normalized pattern: "/foo%20bar/"
  446. assert(ensureMatch("/foo/bar/", "/foo/bar/") is null); // normalized pattern: "/foo/bar/"
  447. //assert(ensureMatch("/foo/bar/", "/foo%2fbar/") !is null);
  448. //assert(ensureMatch("/foo%2fbar/", "/foo%2fbar/") is null); // normalized pattern: "/foo%2Fbar/"
  449. //assert(ensureMatch("/foo%2Fbar/", "/foo%2fbar/") is null); // normalized pattern: "/foo%2Fbar/"
  450. //assert(ensureMatch("/foo%2fbar/", "/foo%2Fbar/") is null);
  451. //assert(ensureMatch("/foo%2fbar/", "/foo/bar/") !is null);
  452. //assert(ensureMatch("/:foo/", "/foo%2Fbar/", ["foo": "foo/bar"]) is null);
  453. assert(ensureMatch("/:foo/", "/foo/bar/") !is null);
  454. assert(ensureMatch("/test", "/tes%74") is null);
  455. }
  456. unittest { // issue #2561
  457. import vibe.http.server : createTestHTTPServerRequest, createTestHTTPServerResponse;
  458. import vibe.inet.url : URL;
  459. import vibe.stream.memory : createMemoryOutputStream;
  460. Route[] routes = [
  461. Route(HTTPMethod.PUT, "/public/devices/commandList"),
  462. Route(HTTPMethod.PUT, "/public/devices/logicStateList"),
  463. Route(HTTPMethod.OPTIONS, "/public/devices/commandList"),
  464. Route(HTTPMethod.OPTIONS, "/public/devices/logicStateList"),
  465. Route(HTTPMethod.PUT, "/public/mnemoschema"),
  466. Route(HTTPMethod.PUT, "/public/static"),
  467. Route(HTTPMethod.PUT, "/public/dynamic"),
  468. Route(HTTPMethod.PUT, "/public/info"),
  469. Route(HTTPMethod.PUT, "/public/info-network"),
  470. Route(HTTPMethod.PUT, "/public/events"),
  471. Route(HTTPMethod.PUT, "/public/eventList"),
  472. Route(HTTPMethod.PUT, "/public/availBatteryModels"),
  473. Route(HTTPMethod.OPTIONS, "/public/availBatteryModels"),
  474. Route(HTTPMethod.OPTIONS, "/public/dynamic"),
  475. Route(HTTPMethod.OPTIONS, "/public/eventList"),
  476. Route(HTTPMethod.OPTIONS, "/public/events"),
  477. Route(HTTPMethod.OPTIONS, "/public/info"),
  478. Route(HTTPMethod.OPTIONS, "/public/info-network"),
  479. Route(HTTPMethod.OPTIONS, "/public/mnemoschema"),
  480. Route(HTTPMethod.OPTIONS, "/public/static"),
  481. Route(HTTPMethod.PUT, "/settings/admin/getinfo"),
  482. Route(HTTPMethod.PUT, "/settings/admin/setconf"),
  483. Route(HTTPMethod.PUT, "/settings/admin/checksetaccess"),
  484. Route(HTTPMethod.OPTIONS, "/settings/admin/checksetaccess"),
  485. Route(HTTPMethod.OPTIONS, "/settings/admin/getinfo"),
  486. Route(HTTPMethod.OPTIONS, "/settings/admin/setconf"),
  487. ];
  488. auto router = new URLRouter;
  489. foreach (r; routes)
  490. router.match(r.method, r.pattern, (req, res) {
  491. res.writeBody("OK");
  492. });
  493. { // make sure unmatched routes are not handled by the router
  494. auto req = createTestHTTPServerRequest(URL("http://localhost/foobar"), HTTPMethod.PUT);
  495. auto res = createTestHTTPServerResponse();
  496. router.handleRequest(req, res);
  497. assert(!res.headerWritten);
  498. }
  499. // ensure all routes are matched
  500. foreach (r; routes) {
  501. auto url = URL("http://localhost"~r.pattern);
  502. auto output = createMemoryOutputStream();
  503. auto req = createTestHTTPServerRequest(url, r.method);
  504. auto res = createTestHTTPServerResponse(output, null, TestHTTPResponseMode.bodyOnly);
  505. router.handleRequest(req, res);
  506. //assert(res.headerWritten);
  507. assert(output.data == "OK");
  508. }
  509. }
  510. /**
  511. Convenience abstraction for a single `URLRouter` route.
  512. See `URLRouter.route` for a usage example.
  513. */
  514. struct URLRoute {
  515. @safe:
  516. URLRouter router;
  517. string path;
  518. ref URLRoute get(Handler)(Handler h) { router.get(path, h); return this; }
  519. ref URLRoute post(Handler)(Handler h) { router.post(path, h); return this; }
  520. ref URLRoute put(Handler)(Handler h) { router.put(path, h); return this; }
  521. ref URLRoute delete_(Handler)(Handler h) { router.delete_(path, h); return this; }
  522. ref URLRoute patch(Handler)(Handler h) { router.patch(path, h); return this; }
  523. ref URLRoute any(Handler)(Handler h) { router.any(path, h); return this; }
  524. ref URLRoute match(Handler)(HTTPMethod method, Handler h) { router.match(method, path, h); return this; }
  525. }
  526. private enum maxRouteParameters = 64;
  527. private struct Route {
  528. HTTPMethod method;
  529. string pattern;
  530. HTTPServerRequestDelegate cb;
  531. }
  532. private string skipPathNode(string str, ref size_t idx)
  533. @safe {
  534. size_t start = idx;
  535. while( idx < str.length && str[idx] != '/' ) idx++;
  536. return str[start .. idx];
  537. }
  538. private string skipPathNode(ref string str)
  539. @safe {
  540. size_t idx = 0;
  541. auto ret = skipPathNode(str, idx);
  542. str = str[idx .. $];
  543. return ret;
  544. }
  545. private struct MatchTree(T) {
  546. @safe:
  547. import std.algorithm : countUntil;
  548. import std.array : array;
  549. private {
  550. alias NodeIndex = uint;
  551. alias TerminalTagIndex = uint;
  552. alias TerminalIndex = ushort;
  553. alias VarIndex = ushort;
  554. struct Node {
  555. TerminalTagIndex terminalsStart; // slice into m_terminalTags
  556. TerminalTagIndex terminalsEnd;
  557. NodeIndex[ubyte.max+1] edges = NodeIndex.max; // character -> index into m_nodes
  558. }
  559. struct TerminalTag {
  560. TerminalIndex index; // index into m_terminals array
  561. VarIndex var = VarIndex.max; // index into Terminal.varNames/varValues or VarIndex.max
  562. }
  563. struct Terminal {
  564. string pattern;
  565. T data;
  566. string[] varNames;
  567. VarIndex[NodeIndex] varMap;
  568. }
  569. Node[] m_nodes; // all nodes as a single array
  570. TerminalTag[] m_terminalTags;
  571. Terminal[] m_terminals;
  572. enum TerminalChar = 0;
  573. bool m_upToDate = false;
  574. }
  575. @property size_t terminalCount() const { return m_terminals.length; }
  576. void addTerminal(string pattern, T data)
  577. {
  578. assert(m_terminals.length < TerminalIndex.max, "Attempt to register too many routes.");
  579. m_terminals ~= Terminal(pattern, data, null, null);
  580. m_upToDate = false;
  581. }
  582. bool match(string text, scope bool delegate(size_t terminal, scope string[] vars) @safe del)
  583. {
  584. // lazily update the match graph
  585. if (!m_upToDate) rebuildGraph();
  586. return doMatch(text, del);
  587. }
  588. const(string)[] getTerminalVarNames(size_t terminal) const { return m_terminals[terminal].varNames; }
  589. ref inout(T) getTerminalData(size_t terminal) inout { return m_terminals[terminal].data; }
  590. void print()
  591. const {
  592. import std.algorithm : map;
  593. import std.array : join;
  594. import std.conv : to;
  595. import std.range : iota;
  596. import std.string : format;
  597. logInfo("Nodes:");
  598. foreach (i, n; m_nodes) {
  599. logInfo(" %s %s", i, m_terminalTags[n.terminalsStart .. n.terminalsEnd]
  600. .map!(t => format("T%s%s", t.index, t.var != VarIndex.max ? "("~m_terminals[t.index].varNames[t.var]~")" : "")).join(" "));
  601. //logInfo(" %s %s-%s", i, n.terminalsStart, n.terminalsEnd);
  602. static string mapChar(ubyte ch) {
  603. if (ch == TerminalChar) return "$";
  604. if (ch >= '0' && ch <= '9') return to!string(cast(dchar)ch);
  605. if (ch >= 'a' && ch <= 'z') return to!string(cast(dchar)ch);
  606. if (ch >= 'A' && ch <= 'Z') return to!string(cast(dchar)ch);
  607. if (ch == '/') return "/";
  608. if (ch == '^') return "^";
  609. return ch.to!string;
  610. }
  611. void printRange(uint node, ubyte from, ubyte to)
  612. {
  613. if (to - from <= 10) logInfo(" %s -> %s", iota(from, cast(uint)to+1).map!(ch => mapChar(cast(ubyte)ch)).join("|"), node);
  614. else logInfo(" %s-%s -> %s", mapChar(from), mapChar(to), node);
  615. }
  616. auto last_to = NodeIndex.max;
  617. ubyte last_ch = 0;
  618. foreach (ch, e; n.edges)
  619. if (e != last_to) {
  620. if (last_to != NodeIndex.max)
  621. printRange(last_to, last_ch, cast(ubyte)(ch-1));
  622. last_ch = cast(ubyte)ch;
  623. last_to = e;
  624. }
  625. if (last_to != NodeIndex.max)
  626. printRange(last_to, last_ch, ubyte.max);
  627. }
  628. }
  629. private bool doMatch(string text, scope bool delegate(size_t terminal, scope string[] vars) @safe del)
  630. const {
  631. string[maxRouteParameters] vars_buf;// = void;
  632. import std.algorithm : canFind;
  633. // first, determine the end node, if any
  634. auto n = matchTerminals(text);
  635. if (!n) return false;
  636. // then, go through the terminals and match their variables
  637. foreach (ref t; m_terminalTags[n.terminalsStart .. n.terminalsEnd]) {
  638. auto term = &m_terminals[t.index];
  639. auto vars = vars_buf[0 .. term.varNames.length];
  640. matchVars(vars, term, text);
  641. if (vars.canFind!(v => v.length == 0)) continue; // all variables must be non-empty to match
  642. if (del(t.index, vars)) return true;
  643. }
  644. return false;
  645. }
  646. /// Given a hexadecimal character in [0-9a-fA-F], convert it to an integer value in [0, 15].
  647. private static uint hexDigit(char ch) @safe nothrow @nogc {
  648. assert((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f'));
  649. if (ch >= '0' && ch <= '9') return ch - '0';
  650. else if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10;
  651. else return ch - 'A' + 10;
  652. }
  653. /// Reads a single character from text, decoding any unreserved percent-encoded character, so
  654. /// that it matches the format used for route matches.
  655. static char nextMatchChar(string text, ref size_t i) {
  656. import std.ascii : isHexDigit;
  657. char ch = text[i];
  658. // See if we have to decode an encoded unreserved character.
  659. if (ch == '%' && i + 2 < text.length && isHexDigit(text[i+1]) && isHexDigit(text[i+2])) {
  660. uint c = hexDigit(text[i+1]) * 16 + hexDigit(text[i+2]);
  661. // Check if we have an encoded unreserved character:
  662. // https://en.wikipedia.org/wiki/Percent-encoding
  663. if (c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
  664. || c == '-' || c == '_' || c == '.' || c == '~') {
  665. // Decode the character before performing route matching.
  666. ch = cast(char) c;
  667. i += 3;
  668. return ch;
  669. }
  670. }
  671. i += 1;
  672. return ch;
  673. }
  674. private inout(Node)* matchTerminals(string text)
  675. inout {
  676. if (!m_nodes.length) return null;
  677. auto n = &m_nodes[0];
  678. // follow the path through the match graph
  679. // Routes match according to their percent-encoded normal form, with reserved-characters
  680. // percent-encoded and unreserved-charcters not percent-encoded.
  681. size_t i = 0;
  682. while (i < text.length) {
  683. char ch = nextMatchChar(text, i);
  684. auto nidx = n.edges[cast(size_t)ch];
  685. if (nidx == NodeIndex.max) return null;
  686. n = &m_nodes[nidx];
  687. }
  688. // finally, find a matching terminal node
  689. auto nidx = n.edges[TerminalChar];
  690. if (nidx == NodeIndex.max) return null;
  691. n = &m_nodes[nidx];
  692. return n;
  693. }
  694. private void matchVars(string[] dst, in Terminal* term, string text)
  695. const {
  696. NodeIndex nidx = 0;
  697. VarIndex activevar = VarIndex.max;
  698. size_t activevarstart;
  699. dst[] = null;
  700. // follow the path through the match graph
  701. size_t i = 0;
  702. while (i < text.length) {
  703. auto var = term.varMap.get(nidx, VarIndex.max);
  704. // detect end of variable
  705. if (var != activevar && activevar != VarIndex.max) {
  706. dst[activevar] = text[activevarstart .. i-1];
  707. activevar = VarIndex.max;
  708. }
  709. // detect beginning of variable
  710. if (var != VarIndex.max && activevar == VarIndex.max) {
  711. activevar = var;
  712. activevarstart = i;
  713. }
  714. char ch = nextMatchChar(text, i);
  715. nidx = m_nodes[nidx].edges[cast(ubyte)ch];
  716. assert(nidx != NodeIndex.max);
  717. }
  718. // terminate any active varible with the end of the input string or with the last character
  719. auto var = term.varMap.get(nidx, VarIndex.max);
  720. if (activevar != VarIndex.max) dst[activevar] = text[activevarstart .. (var == activevar ? $ : $-1)];
  721. }
  722. private void rebuildGraph()
  723. @trusted {
  724. import std.array : appender;
  725. import std.conv : to;
  726. if (m_upToDate) return;
  727. m_upToDate = true;
  728. m_nodes = null;
  729. m_terminalTags = null;
  730. if (!m_terminals.length) return;
  731. MatchGraphBuilder builder;
  732. foreach (i, ref t; m_terminals) {
  733. t.varNames = builder.insert(t.pattern, i.to!TerminalIndex);
  734. assert(t.varNames.length <= VarIndex.max, "Too many variables in route.");
  735. }
  736. //builder.print();
  737. builder.disambiguate();
  738. auto nodemap = new NodeIndex[builder.m_nodes.length];
  739. nodemap[] = NodeIndex.max;
  740. auto nodes = appender!(Node[]);
  741. nodes.reserve(1024);
  742. auto termtags = appender!(TerminalTag[]);
  743. termtags.reserve(1024);
  744. NodeIndex process(NodeIndex n)
  745. {
  746. import std.algorithm : canFind;
  747. if (nodemap[n] != NodeIndex.max) return nodemap[n];
  748. auto nmidx = cast(NodeIndex)nodes.data.length;
  749. nodemap[n] = nmidx;
  750. nodes.put(Node.init);
  751. Node nn;
  752. nn.terminalsStart = termtags.data.length.to!TerminalTagIndex;
  753. foreach (t; builder.m_nodes[n].terminals) {
  754. auto var = cast(VarIndex)t.var;
  755. assert(!termtags.data[nn.terminalsStart .. $].canFind!(u => u.index == t.index && u.var == var));
  756. termtags ~= TerminalTag(cast(TerminalIndex)t.index, var);
  757. if (var != VarIndex.max)
  758. m_terminals[t.index].varMap[nmidx] = var;
  759. }
  760. nn.terminalsEnd = termtags.data.length.to!TerminalTagIndex;
  761. foreach (ch, targets; builder.m_nodes[n].edges)
  762. foreach (to; builder.m_edgeEntries.getItems(targets))
  763. nn.edges[ch] = process(to);
  764. nodes.data[nmidx] = nn;
  765. return nmidx;
  766. }
  767. assert(builder.m_edgeEntries.hasLength(builder.m_nodes[0].edges['^'], 1),
  768. "Graph must be disambiguated before purging.");
  769. process(builder.m_edgeEntries.getItems(builder.m_nodes[0].edges['^']).front);
  770. m_nodes = nodes.data;
  771. m_terminalTags = termtags.data;
  772. logDebug("Match tree has %s (of %s in the builder) nodes, %s terminals", m_nodes.length, builder.m_nodes.length, m_terminals.length);
  773. }
  774. }
  775. unittest {
  776. import std.string : format;
  777. MatchTree!int m;
  778. void testMatch(string str, size_t[] terms, string[] vars)
  779. {
  780. size_t[] mterms;
  781. string[] mvars;
  782. m.match(str, (t, scope vals) {
  783. mterms ~= t;
  784. mvars ~= vals;
  785. return false;
  786. });
  787. assert(mterms == terms, format("Mismatched terminals: %s (expected %s)", mterms, terms));
  788. assert(mvars == vars, format("Mismatched variables; %s (expected %s)", mvars, vars));
  789. }
  790. m.addTerminal("a", 0);
  791. m.addTerminal("b", 0);
  792. m.addTerminal("ab", 0);
  793. m.rebuildGraph();
  794. assert(m.getTerminalVarNames(0) == []);
  795. assert(m.getTerminalVarNames(1) == []);
  796. assert(m.getTerminalVarNames(2) == []);
  797. testMatch("a", [0], []);
  798. testMatch("ab", [2], []);
  799. testMatch("abc", [], []);
  800. testMatch("b", [1], []);
  801. m = MatchTree!int.init;
  802. m.addTerminal("ab", 0);
  803. m.addTerminal("a*", 0);
  804. m.rebuildGraph();
  805. assert(m.getTerminalVarNames(0) == []);
  806. assert(m.getTerminalVarNames(1) == []);
  807. testMatch("a", [1], []);
  808. testMatch("ab", [0, 1], []);
  809. testMatch("abc", [1], []);
  810. m = MatchTree!int.init;
  811. m.addTerminal("ab", 0);
  812. m.addTerminal("a:var", 0);
  813. m.rebuildGraph();
  814. assert(m.getTerminalVarNames(0) == []);
  815. assert(m.getTerminalVarNames(1) == ["var"], format("%s", m.getTerminalVarNames(1)));
  816. testMatch("a", [], []); // vars may not be empty
  817. testMatch("ab", [0, 1], ["b"]);
  818. testMatch("abc", [1], ["bc"]);
  819. m = MatchTree!int.init;
  820. m.addTerminal(":var1/:var2", 0);
  821. m.addTerminal("a/:var3", 0);
  822. m.addTerminal(":var4/b", 0);
  823. m.rebuildGraph();
  824. assert(m.getTerminalVarNames(0) == ["var1", "var2"]);
  825. assert(m.getTerminalVarNames(1) == ["var3"]);
  826. assert(m.getTerminalVarNames(2) == ["var4"]);
  827. testMatch("a", [], []);
  828. testMatch("a/a", [0, 1], ["a", "a", "a"]);
  829. testMatch("a/b", [0, 1, 2], ["a", "b", "b", "a"]);
  830. testMatch("a/bc", [0, 1], ["a", "bc", "bc"]);
  831. testMatch("ab/b", [0, 2], ["ab", "b", "ab"]);
  832. testMatch("ab/bc", [0], ["ab", "bc"]);
  833. m = MatchTree!int.init;
  834. m.addTerminal(":var1/", 0);
  835. m.rebuildGraph();
  836. assert(m.getTerminalVarNames(0) == ["var1"]);
  837. testMatch("ab/", [0], ["ab"]);
  838. testMatch("ab", [], []);
  839. testMatch("/ab", [], []);
  840. testMatch("a/b", [], []);
  841. testMatch("ab//", [], []);
  842. }
  843. private struct MatchGraphBuilder {
  844. @safe:
  845. import std.container.array : Array;
  846. import std.array : array;
  847. import std.algorithm : filter;
  848. import std.string : format;
  849. alias NodeIndex = uint;
  850. alias TerminalIndex = ushort;
  851. alias VarIndex = ushort;
  852. alias NodeSet = LinkedSetBacking!NodeIndex.Handle;
  853. private {
  854. enum TerminalChar = 0;
  855. struct TerminalTag {
  856. TerminalIndex index;
  857. VarIndex var = VarIndex.max;
  858. }
  859. struct Node {
  860. Array!TerminalTag terminals;
  861. NodeSet[ubyte.max+1] edges;
  862. }
  863. Array!Node m_nodes;
  864. LinkedSetBacking!NodeIndex m_edgeEntries;
  865. }
  866. @disable this(this);
  867. string[] insert(string pattern, TerminalIndex terminal)
  868. {
  869. import std.algorithm : canFind;
  870. auto full_pattern = pattern;
  871. string[] vars;
  872. if (!m_nodes.length) addNode();
  873. // create start node and connect to zero node
  874. auto nidx = addNode();
  875. addEdge(0, nidx, '^', terminal);
  876. while (pattern.length) {
  877. auto ch = pattern[0];
  878. if (ch == '*') {
  879. assert(pattern.length == 1, "Asterisk is only allowed at the end of a pattern: "~full_pattern);
  880. pattern = null;
  881. foreach (v; ubyte.min .. ubyte.max+1) {
  882. if (v == TerminalChar) continue;
  883. addEdge(nidx, nidx, cast(ubyte)v, terminal);
  884. }
  885. } else if (ch == ':') {
  886. pattern = pattern[1 .. $];
  887. auto name = skipPathNode(pattern);
  888. assert(name.length > 0, "Missing placeholder name: "~full_pattern);
  889. assert(!vars.canFind(name), "Duplicate placeholder name ':"~name~"': '"~full_pattern~"'");
  890. auto varidx = cast(VarIndex)vars.length;
  891. vars ~= name;
  892. assert(!pattern.length || (pattern[0] != '*' && pattern[0] != ':'),
  893. "Cannot have two placeholders directly follow each other.");
  894. foreach (v; ubyte.min .. ubyte.max+1) {
  895. if (v == TerminalChar || v == '/') continue;
  896. addEdge(nidx, nidx, cast(ubyte)v, terminal, varidx);
  897. }
  898. } else {
  899. nidx = addEdge(nidx, ch, terminal);
  900. pattern = pattern[1 .. $];
  901. }
  902. }
  903. addEdge(nidx, TerminalChar, terminal);
  904. return vars;
  905. }
  906. void disambiguate()
  907. @trusted {
  908. import std.algorithm : map, sum;
  909. import std.array : appender, join;
  910. //logInfo("Disambiguate with %s initial nodes", m_nodes.length);
  911. if (!m_nodes.length) return;
  912. import vibe.container.hashmap : HashMap;
  913. HashMap!(LinkedSetHash, NodeIndex) combined_nodes;
  914. Array!bool visited;
  915. visited.length = m_nodes.length * 2;
  916. Stack!NodeIndex node_stack;
  917. node_stack.reserve(m_nodes.length);
  918. node_stack.push(0);
  919. while (!node_stack.empty) {
  920. auto n = node_stack.pop();
  921. while (n >= visited.length) visited.length = visited.length * 2;
  922. if (visited[n]) continue;
  923. //logInfo("Disambiguate %s (stack=%s)", n, node_stack.fill);
  924. visited[n] = true;
  925. foreach (ch; ubyte.min .. ubyte.max+1) {
  926. auto chnodes = m_nodes[n].edges[ch];
  927. LinkedSetHash chhash = m_edgeEntries.getHash(chnodes);
  928. // handle trivial cases
  929. if (m_edgeEntries.hasMaxLength(chnodes, 1))
  930. continue;
  931. // generate combined state for ambiguous edges
  932. if (auto pn = () @trusted { return chhash in combined_nodes; } ()) {
  933. m_nodes[n].edges[ch] = m_edgeEntries.create(*pn);
  934. assert(m_edgeEntries.hasLength(m_nodes[n].edges[ch], 1));
  935. continue;
  936. }
  937. // for new combinations, create a new node
  938. NodeIndex ncomb = addNode();
  939. combined_nodes[chhash] = ncomb;
  940. // write all edges
  941. size_t idx = 0;
  942. foreach (to_ch; ubyte.min .. ubyte.max+1) {
  943. auto e = &m_nodes[ncomb].edges[to_ch];
  944. foreach (chn; m_edgeEntries.getItems(chnodes))
  945. m_edgeEntries.insert(e, m_edgeEntries.getItems(m_nodes[chn].edges[to_ch]));
  946. }
  947. // add terminal indices
  948. foreach (chn; m_edgeEntries.getItems(chnodes))
  949. foreach (t; m_nodes[chn].terminals)
  950. addTerminal(ncomb, t.index, t.var);
  951. foreach (i; 1 .. m_nodes[ncomb].terminals.length)
  952. assert(m_nodes[ncomb].terminals[0] != m_nodes[ncomb].terminals[i]);
  953. m_nodes[n].edges[ch] = m_edgeEntries.create(ncomb);
  954. assert(m_edgeEntries.hasLength(m_nodes[n].edges[ch], 1));
  955. }
  956. // process nodes recursively
  957. foreach (ch; ubyte.min .. ubyte.max+1) {
  958. // should only have single out-edges now
  959. assert(m_edgeEntries.hasMaxLength(m_nodes[n].edges[ch], 1));
  960. foreach (e; m_edgeEntries.getItems(m_nodes[n].edges[ch]))
  961. node_stack.push(e);
  962. }
  963. }
  964. import std.algorithm.sorting : sort;
  965. foreach (ref n; m_nodes)
  966. n.terminals[].sort!((a, b) => a.index < b.index)();
  967. debug logDebug("Disambiguate done: %s nodes, %s max stack size", m_nodes.length, node_stack.maxSize);
  968. }
  969. void print()
  970. const @trusted {
  971. import std.algorithm : map;
  972. import std.array : join;
  973. import std.conv : to;
  974. import std.string : format;
  975. logInfo("Nodes:");
  976. size_t i = 0;
  977. foreach (n; m_nodes) {
  978. string mapChar(ubyte ch) {
  979. if (ch == TerminalChar) return "$";
  980. if (ch >= '0' && ch <= '9') return to!string(cast(dchar)ch);
  981. if (ch >= 'a' && ch <= 'z') return to!string(cast(dchar)ch);
  982. if (ch >= 'A' && ch <= 'Z') return to!string(cast(dchar)ch);
  983. if (ch == '^') return "^";
  984. if (ch == '/') return "/";
  985. return format("$%s", ch);
  986. }
  987. logInfo(" %s: %s", i, n.terminals[].map!(t => t.var != VarIndex.max ? format("T%s(%s)", t.index, t.var) : format("T%s", t.index)).join(" "));
  988. ubyte first_char;
  989. LinkedSetHash list_hash;
  990. NodeSet list;
  991. void printEdges(ubyte last_char) {
  992. if (!list.empty) {
  993. string targets;
  994. foreach (tn; m_edgeEntries.getItems(list))
  995. targets ~= format(" %s", tn);
  996. if (targets.length > 0)
  997. logInfo(" [%s ... %s] -> %s", mapChar(first_char), mapChar(last_char), targets);
  998. }
  999. }
  1000. foreach (ch, tnodes; n.edges) {
  1001. auto h = m_edgeEntries.getHash(tnodes);
  1002. if (h != list_hash) {
  1003. printEdges(cast(ubyte)(ch-1));
  1004. list_hash = h;
  1005. list = tnodes;
  1006. first_char = cast(ubyte)ch;
  1007. }
  1008. }
  1009. printEdges(ubyte.max);
  1010. i++;
  1011. }
  1012. }
  1013. private void addEdge(NodeIndex from, NodeIndex to, ubyte ch, TerminalIndex terminal, VarIndex var = VarIndex.max)
  1014. @trusted {
  1015. m_edgeEntries.insert(&m_nodes[from].edges[ch], to);
  1016. addTerminal(to, terminal, var);
  1017. }
  1018. private NodeIndex addEdge(NodeIndex from, ubyte ch, TerminalIndex terminal, VarIndex var = VarIndex.max)
  1019. @trusted {
  1020. import std.algorithm : canFind;
  1021. import std.string : format;
  1022. if (!m_nodes[from].edges[ch].empty)
  1023. assert(false, format("%s is in %s", ch, m_nodes[from].edges[]));
  1024. auto nidx = addNode();
  1025. addEdge(from, nidx, ch, terminal, var);
  1026. return nidx;
  1027. }
  1028. private void addTerminal(NodeIndex node, TerminalIndex terminal, VarIndex var)
  1029. @trusted {
  1030. foreach (ref t; m_nodes[node].terminals) {
  1031. if (t.index == terminal) {
  1032. if (t.var != VarIndex.max && t.var != var)
  1033. assert(false, format("Ambiguous route var match!? %s vs %s", t.var, var));
  1034. t.var = var;
  1035. return;
  1036. }
  1037. }
  1038. m_nodes[node].terminals ~= TerminalTag(terminal, var);
  1039. }
  1040. private NodeIndex addNode()
  1041. @trusted {
  1042. assert(m_nodes.length <= 1_000_000_000, "More than 1B nodes in tree!?");
  1043. auto idx = cast(NodeIndex)m_nodes.length;
  1044. m_nodes ~= Node.init;
  1045. return idx;
  1046. }
  1047. }
  1048. /** Used to store and manipulate multiple linked lists within a single array
  1049. based storage.
  1050. */
  1051. struct LinkedSetBacking(T) {
  1052. import std.container.array : Array;
  1053. import std.range : isInputRange;
  1054. static struct Handle {
  1055. uint index = uint.max;
  1056. @property bool empty() const { return index == uint.max; }
  1057. }
  1058. private {
  1059. static struct Entry {
  1060. uint next;
  1061. T value;
  1062. }
  1063. Array!Entry m_storage;
  1064. static struct Range {
  1065. private {
  1066. Array!Entry* backing;
  1067. uint index;
  1068. }
  1069. @property bool empty() const { return index == uint.max; }
  1070. @property ref const(T) front() const { return (*backing)[index].value; }
  1071. void popFront()
  1072. {
  1073. index = (*backing)[index].next;
  1074. }
  1075. }
  1076. }
  1077. @property Handle emptySet() { return Handle.init; }
  1078. Handle create(scope T[] items...)
  1079. {
  1080. Handle ret;
  1081. foreach (i; items)
  1082. ret.index = createNode(i, ret.index);
  1083. return ret;
  1084. }
  1085. void insert(Handle* h, T value)
  1086. {
  1087. /*foreach (c; getItems(*h))
  1088. if (value == c)
  1089. return;*/
  1090. h.index = createNode(value, h.index);
  1091. }
  1092. void insert(R)(Handle* h, R items)
  1093. if (isInputRange!R)
  1094. {
  1095. foreach (itm; items)
  1096. insert(h, itm);
  1097. }
  1098. LinkedSetHash getHash(Handle sh)
  1099. const {
  1100. // NOTE: the returned hash is order independent, to avoid bogus
  1101. // mismatches when comparing lists of different order
  1102. LinkedSetHash ret = linkedSetHashOf(null);
  1103. while (sh != Handle.init) {
  1104. auto h = linkedSetHashOf(cast(const(ubyte)[])(&m_storage[sh.index].value)[0 .. 1]);
  1105. foreach (i; 0 .. ret.length) ret[i] ^= h[i];
  1106. sh.index = m_storage[sh.index].next;
  1107. }
  1108. return ret;
  1109. }
  1110. auto getItems(Handle sh) { return Range(&m_storage, sh.index); }
  1111. auto getItems(Handle sh) const { return Range(&(cast()this).m_storage, sh.index); }
  1112. bool hasMaxLength(Handle sh, size_t l)
  1113. const {
  1114. uint idx = sh.index;
  1115. do {
  1116. if (idx == uint.max) return true;
  1117. idx = m_storage[idx].next;
  1118. } while (l-- > 0);
  1119. return false;
  1120. }
  1121. bool hasLength(Handle sh, size_t l)
  1122. const {
  1123. uint idx = sh.index;
  1124. while (l-- > 0) {
  1125. if (idx == uint.max) return false;
  1126. idx = m_storage[idx].next;
  1127. }
  1128. return idx == uint.max;
  1129. }
  1130. private uint createNode(ref T val, uint next)
  1131. {
  1132. auto id = cast(uint)m_storage.length;
  1133. m_storage ~= Entry(next, val);
  1134. return id;
  1135. }
  1136. }
  1137. unittest {
  1138. import std.algorithm.comparison : equal;
  1139. LinkedSetBacking!int b;
  1140. auto s = b.emptySet;
  1141. assert(s.empty);
  1142. assert(b.getItems(s).empty);
  1143. s = b.create(3, 5, 7);
  1144. assert(b.getItems(s).equal([7, 5, 3]));
  1145. assert(!b.hasLength(s, 2));
  1146. assert(b.hasLength(s, 3));
  1147. assert(!b.hasLength(s, 4));
  1148. assert(!b.hasMaxLength(s, 2));
  1149. assert(b.hasMaxLength(s, 3));
  1150. assert(b.hasMaxLength(s, 4));
  1151. auto h = b.getHash(s);
  1152. assert(h != b.getHash(b.emptySet));
  1153. s = b.create(5, 3, 7);
  1154. assert(b.getHash(s) == h);
  1155. assert(b.getItems(s).equal([7, 3, 5]));
  1156. b.insert(&s, 11);
  1157. assert(b.hasLength(s, 4));
  1158. assert(b.getHash(s) != h);
  1159. assert(b.getItems(s).equal([11, 7, 3, 5]));
  1160. }
  1161. alias LinkedSetHasher = MurmurHash3!(128, 64);
  1162. alias LinkedSetHash = ulong[16/ulong.sizeof];
  1163. LinkedSetHash linkedSetHashOf(scope const(ubyte)[] bytes)
  1164. {
  1165. LinkedSetHasher h;
  1166. h.start();
  1167. h.put(bytes);
  1168. return cast(LinkedSetHash)h.finish();
  1169. }
  1170. private struct Stack(E)
  1171. {
  1172. import std.range : isInputRange;
  1173. private {
  1174. E[] m_storage;
  1175. size_t m_fill;
  1176. debug size_t m_maxFill;
  1177. }
  1178. @property bool empty() const { return m_fill == 0; }
  1179. @property size_t fill() const { return m_fill; }
  1180. debug @property size_t maxSize() const { return m_maxFill; }
  1181. void reserve(size_t amt)
  1182. {
  1183. auto minsz = m_fill + amt;
  1184. if (m_storage.length < minsz) {
  1185. auto newlength = 64;
  1186. while (newlength < minsz) newlength *= 2;
  1187. m_storage.length = newlength;
  1188. }
  1189. }
  1190. void push(E el)
  1191. {
  1192. reserve(1);
  1193. m_storage[m_fill++] = el;
  1194. debug if (m_fill > m_maxFill) m_maxFill = m_fill;
  1195. }
  1196. void push(R)(R els)
  1197. if (isInputRange!R)
  1198. {
  1199. reserve(els.length);
  1200. foreach (el; els)
  1201. m_storage[m_fill++] = el;
  1202. debug if (m_fill > m_maxFill) m_maxFill = m_fill;
  1203. }
  1204. E pop()
  1205. {
  1206. assert(!empty, "Stack underflow.");
  1207. return m_storage[--m_fill];
  1208. }
  1209. }