From e88e680ed7bd2799cdfd6f627dfc85f064dee94c Mon Sep 17 00:00:00 2001 From: Ruben Taelman Date: Mon, 31 Aug 2020 09:12:35 +0200 Subject: [PATCH] feat: add support for parsing more RDF formats using rdf-parse --- bin/server.ts | 4 +- index.ts | 1 + package-lock.json | 434 +++++++++++++++++- package.json | 1 + src/storage/conversion/RdfToQuadConverter.ts | 41 ++ .../conversion/RdfToQuadConverter.test.ts | 86 ++++ 6 files changed, 563 insertions(+), 4 deletions(-) create mode 100644 src/storage/conversion/RdfToQuadConverter.ts create mode 100644 test/unit/storage/conversion/RdfToQuadConverter.test.ts diff --git a/bin/server.ts b/bin/server.ts index 042b71779..7dbcf6e6d 100644 --- a/bin/server.ts +++ b/bin/server.ts @@ -6,6 +6,7 @@ import { BasePermissionsExtractor, CompositeAsyncHandler, ExpressHttpServer, + RdfToQuadConverter, HttpRequest, PatchingStore, QuadToTurtleConverter, @@ -30,7 +31,6 @@ import { SimpleTargetExtractor, SingleThreadedResourceLocker, SparqlPatchPermissionsExtractor, - TurtleToQuadConverter, UrlContainerManager, } from '..'; @@ -63,7 +63,7 @@ const permissionsExtractor = new CompositeAsyncHandler([ // Will have to see how to best handle this const store = new SimpleResourceStore(runtimeConfig); const converter = new CompositeAsyncHandler([ - new TurtleToQuadConverter(), + new RdfToQuadConverter(), new QuadToTurtleConverter(), ]); const convertingStore = new RepresentationConvertingStore(store, converter); diff --git a/index.ts b/index.ts index 5425865dd..5e8ffb307 100644 --- a/index.ts +++ b/index.ts @@ -64,6 +64,7 @@ export * from './src/server/HttpRequest'; export * from './src/server/HttpResponse'; // Storage/Conversion +export * from './src/storage/conversion/RdfToQuadConverter'; export * from './src/storage/conversion/QuadToTurtleConverter'; export * from './src/storage/conversion/RepresentationConverter'; export * from './src/storage/conversion/TurtleToQuadConverter'; diff --git a/package-lock.json b/package-lock.json index c9b5c0186..5c9bd97d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -411,6 +411,155 @@ "minimist": "^1.2.0" } }, + "@comunica/actor-abstract-mediatyped": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-abstract-mediatyped/-/actor-abstract-mediatyped-1.15.0.tgz", + "integrity": "sha512-rB0DnIuiZKGqAd6JmcXdajmgDPWzffqCqLyAp2A967NRN1NlPbPnfwkCJBHVehdcyT69dAaEkHoHDZpbwOVjHg==" + }, + "@comunica/actor-http-native": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@comunica/actor-http-native/-/actor-http-native-1.16.2.tgz", + "integrity": "sha512-KhmYTKOmf4QZwd8CVy4f4o6d/LVzBy19JmpOIP1vLcOTAVOXg6KXDT+NEVSoDIc9T5fpKDh91bWuukptUqVjig==", + "requires": { + "@types/parse-link-header": "^1.0.0", + "cross-fetch": "^3.0.5", + "follow-redirects": "^1.5.1", + "parse-link-header": "^1.0.1" + } + }, + "@comunica/actor-rdf-parse-html": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-html/-/actor-rdf-parse-html-1.15.0.tgz", + "integrity": "sha512-eC43jtLqY3wfA6aRim83l+3NfoMQCZv/lPP2MvqxY04/Rk77DdqKtKS2GfYrIBc8aqDOoreVgiBQG5TzPzm5Ew==", + "requires": { + "@types/rdf-js": "^3.0.0", + "htmlparser2": "^4.0.0" + } + }, + "@comunica/actor-rdf-parse-html-rdfa": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-html-rdfa/-/actor-rdf-parse-html-rdfa-1.15.0.tgz", + "integrity": "sha512-1rA7YCjY1v7/1oRKzthVw633HZmHux96SQW5j+FIJTvX4GoBvf1seS082HY7WQGRpZVgLxOUxtAOQcp7zT4dHA==", + "requires": { + "rdfa-streaming-parser": "^1.1.1" + } + }, + "@comunica/actor-rdf-parse-html-script": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-html-script/-/actor-rdf-parse-html-script-1.16.1.tgz", + "integrity": "sha512-IYagk0RzIQEzj/78KvoICDgOiPnwvj44S/PH2ha++o9Se0FgTxRjHrWTluETiczOcvrV54TeVh0YDYJck+xz/w==", + "requires": { + "@types/rdf-js": "^3.0.0", + "relative-to-absolute-iri": "^1.0.5" + } + }, + "@comunica/actor-rdf-parse-jsonld": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-jsonld/-/actor-rdf-parse-jsonld-1.16.0.tgz", + "integrity": "sha512-LtAIv75bhHe2teN0vjWZ6EkjwnK6Yw56Q1ZuxgrWZLGrbdRF6YZuXglW+IqWeZ45tIUqhweS83k17/9fR3UK5w==", + "requires": { + "@types/rdf-js": "^3.0.0", + "jsonld-streaming-parser": "^2.0.2", + "stream-to-string": "^1.2.0" + } + }, + "@comunica/actor-rdf-parse-n3": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-n3/-/actor-rdf-parse-n3-1.15.0.tgz", + "integrity": "sha512-YWk7XSDN8GDOFL+u5PtadZmIUzAh5ZUYB/LwXLENeymIgWEaSvJo5H4QqdGmnJFArlgXX2Thk8jTvbtubiNTvw==", + "requires": { + "@types/n3": "^1.4.2", + "n3": "^1.0.0" + }, + "dependencies": { + "@types/n3": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/@types/n3/-/n3-1.4.4.tgz", + "integrity": "sha512-xsWfwyDh0uAH0CXvwqe9vb2UEDafMjRez/pB7yZwbWpd9Olw2wdxaL32FtdHjmmFE6b9i+j249JfRyZnvWkoqg==", + "requires": { + "@types/node": "*", + "@types/rdf-js": "*" + } + } + } + }, + "@comunica/actor-rdf-parse-rdfxml": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-rdfxml/-/actor-rdf-parse-rdfxml-1.15.0.tgz", + "integrity": "sha512-Rec0dnaTW95Mx91cfQPBkDkgEhoFd9J36FtiJAotLFPOiXp4YsTEZGQNSODTbhchfaTS6HDTFnETQ6GbmutaCQ==", + "requires": { + "rdfxml-streaming-parser": "^1.1.0" + } + }, + "@comunica/actor-rdf-parse-xml-rdfa": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/actor-rdf-parse-xml-rdfa/-/actor-rdf-parse-xml-rdfa-1.15.0.tgz", + "integrity": "sha512-34CI/f/JQTEfrnxyqB7Y+EhkhW0tTMsHoC8yu+Y6AtSVdj2FYkSR+GmAk9F65JVnm/RC9uh1T7yT5yJMVvmgiQ==", + "requires": { + "rdfa-streaming-parser": "^1.1.1" + } + }, + "@comunica/bus-http": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/@comunica/bus-http/-/bus-http-1.16.0.tgz", + "integrity": "sha512-3fWJA3Yh7Wj4clabg141FgAH+m8/InZ+BVPr7BqhO+7Jg0tXbvGDfai8N0SXWL7rCMkWjlPYkBHkzpQ8Dk0HAg==", + "requires": { + "is-stream": "^2.0.0", + "web-streams-node": "^0.4.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + } + } + }, + "@comunica/bus-init": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/bus-init/-/bus-init-1.15.0.tgz", + "integrity": "sha512-HwVTEznx2H7GvcfQAREz52QF8xXT1AqxkJDnI9vTeGvLpWrM+LVOAJZxBcYFFK3X3bEhTsPfDVzikziMGqZuZw==" + }, + "@comunica/bus-rdf-parse": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/bus-rdf-parse/-/bus-rdf-parse-1.15.0.tgz", + "integrity": "sha512-6cGGUBgorkJ6hr5U235qSrlK1m7rPpIM8W6o9kbRAw149AKap1XFF3OXpAYqat2eB4Cbs1J2PP9wg1/s/uOsww==", + "requires": { + "@comunica/actor-abstract-mediatyped": "^1.15.0", + "@types/rdf-js": "^3.0.0" + } + }, + "@comunica/bus-rdf-parse-html": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/bus-rdf-parse-html/-/bus-rdf-parse-html-1.15.0.tgz", + "integrity": "sha512-7F/kDrNf9X//IrO/CK4LpwM5f+mFHLa/wsPc2RubyhiFN3P3KWU6NWxjDJRT9yP85EmW4KknXWwF8AOTvbKF1A==", + "requires": { + "@types/rdf-js": "^3.0.0" + } + }, + "@comunica/core": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/core/-/core-1.15.0.tgz", + "integrity": "sha512-Mg/fpufyK4hVKX2vgP8oapu4ZpyugNRxyI/1AX7zlhSkYpE+ldK5YQaGG/sb+fq+s2sAtE1Ocd+thCgF4wv+8w==", + "requires": { + "immutable": "^3.8.2" + } + }, + "@comunica/mediator-combine-union": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/mediator-combine-union/-/mediator-combine-union-1.15.0.tgz", + "integrity": "sha512-TPzfXuKRVBEpjb+/S8v2C8uvgi8BTjtg1htL/qY/xI2fYqRcLZGl+D5MZEESmwpmm19cBfg87CbF+ROn6L3+IQ==" + }, + "@comunica/mediator-number": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/mediator-number/-/mediator-number-1.15.0.tgz", + "integrity": "sha512-d1jwHVN6rELFzEPb+CDqEjItAJ4/AqiP1Vp+Z/PBucLjM9EEonQMjQzqrkRV/Oy6cBXfSQDhXuSvftfl/zUCjg==" + }, + "@comunica/mediator-race": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@comunica/mediator-race/-/mediator-race-1.15.0.tgz", + "integrity": "sha512-MVESJnkgSoPaCkNNN8RDm0B06d0JNL4QOvRZFuFETY4/D+OwIpcEf0EGButQsDUlelAk5n/sFssLMqo2tw65SA==" + }, "@istanbuljs/load-nyc-config": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.0.0.tgz", @@ -841,6 +990,16 @@ "@types/node": "*" } }, + "@types/http-link-header": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/http-link-header/-/http-link-header-1.0.1.tgz", + "integrity": "sha512-5h+Pqs4EHoMkY/fLva7XsYmh9IVQghQ6uWWil1FGCNI0WqjhI4g20doYsbT4kJ/G3GkAlQca4AIc9OexdUnzkg==" + }, + "@types/isomorphic-fetch": { + "version": "0.0.35", + "resolved": "https://registry.npmjs.org/@types/isomorphic-fetch/-/isomorphic-fetch-0.0.35.tgz", + "integrity": "sha512-DaZNUvLDCAnCTjgwxgiL1eQdxIKEpNLOlTNtAgnZc50bG2copGhRrFN9/PxPBuJe+tZVLCbQ7ls0xveXVRPkvw==" + }, "@types/istanbul-lib-coverage": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz", @@ -984,6 +1143,11 @@ "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", "dev": true }, + "@types/parse-link-header": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-link-header/-/parse-link-header-1.0.0.tgz", + "integrity": "sha512-fCA3btjE7QFeRLfcD0Sjg+6/CnmC66HpMBoRfRzd2raTaWMJV21CCZ0LO8MOqf8onl5n0EPfjq4zDhbyX8SVwA==" + }, "@types/prettier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.0.tgz", @@ -1740,6 +1904,11 @@ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, + "canonicalize": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/canonicalize/-/canonicalize-1.0.3.tgz", + "integrity": "sha512-QWAGweNicWIXzcl7skvUZQ/ArdecS8fOeudnjIU0LYqSdTOSBSap+0VPMas4u11cW3a9sN5AN/aJHQUGfdWLCw==" + }, "capture-exit": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/capture-exit/-/capture-exit-2.0.0.tgz", @@ -2044,6 +2213,14 @@ "request": "^2.88.2" } }, + "cross-fetch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.5.tgz", + "integrity": "sha512-FFLcLtraisj5eteosnX1gf01qYDCOc4fDy0+euOt8Kn9YBY2NtXL/pCoYPavw24NIQkQqm5ZOLsGD5Zzj0gyew==", + "requires": { + "node-fetch": "2.6.0" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -2269,6 +2446,21 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.0.1.tgz", + "integrity": "sha512-1Aj1Qy3YLbdslkI75QEOfdp9TkQ3o8LRISAzxOibjBs/xWwr1WxZFOQphFkZuepHFGo+kB8e5FVJSS0faAJ4Rw==", + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^3.0.0", + "entities": "^2.0.0" + } + }, + "domelementtype": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.0.1.tgz", + "integrity": "sha512-5HOHUDsYZWV8FGWN0Njbr/Rn7f/eWSQi1v7+HsUVwXgn8nWWlL64zKDkS0n8ZmQ3mlWOMuXOnR+7Nx/5tMO5AQ==" + }, "domexception": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", @@ -2286,6 +2478,24 @@ } } }, + "domhandler": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-3.0.0.tgz", + "integrity": "sha512-eKLdI5v9m67kbXQbJSNn1zjh0SDzvzWVWtX+qEI3eMjZw8daH9k8rlj1FZY9memPwjiskQFbe7vHVVJIAqoEhw==", + "requires": { + "domelementtype": "^2.0.1" + } + }, + "domutils": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.2.0.tgz", + "integrity": "sha512-0haAxVr1PR0SqYwCH7mxMpHZUwjih9oPPedqpR/KufsnxPyZ9dyVw1R5093qnJF3WXSbjBkdzRWLw/knJV/fAg==", + "requires": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.0.1", + "domhandler": "^3.0.0" + } + }, "dot-prop": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", @@ -2326,6 +2536,24 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "requires": { + "iconv-lite": "^0.6.2" + }, + "dependencies": { + "iconv-lite": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz", + "integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -2335,6 +2563,11 @@ "once": "^1.4.0" } }, + "entities": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz", + "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -3655,6 +3888,11 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -3949,12 +4187,28 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "htmlparser2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-4.1.0.tgz", + "integrity": "sha512-4zDq1a1zhE4gQso/c5LP1OtrhYTncXNSpvJYtWJBtXAETPlMfi3IFNjGuQbYLuVY4ZR0QMqRVvo4Pdy9KLyP8Q==", + "requires": { + "domelementtype": "^2.0.1", + "domhandler": "^3.0.0", + "domutils": "^2.0.0", + "entities": "^2.0.0" + } + }, "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", "dev": true }, + "http-link-header": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/http-link-header/-/http-link-header-1.0.2.tgz", + "integrity": "sha512-z6YOZ8ZEnejkcCWlGZzYXNa6i+ZaTfiTg3WhlV/YvnNya3W/RbX1bMVUMTuCrg/DrtTCQxaFCkXCz4FtLpcebg==" + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -4010,6 +4264,11 @@ "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=", "dev": true }, + "immutable": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz", + "integrity": "sha1-wkOZUUVbs5kT2vKBN28VMOEErfM=" + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -4333,8 +4592,7 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", - "dev": true + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, "is-string": { "version": "1.0.5", @@ -4397,6 +4655,26 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, + "isomorphic-fetch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", + "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", + "requires": { + "node-fetch": "^1.0.1", + "whatwg-fetch": ">=0.10.0" + }, + "dependencies": { + "node-fetch": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "requires": { + "encoding": "^0.1.11", + "is-stream": "^1.0.1" + } + } + } + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -5081,6 +5359,46 @@ "minimist": "^1.2.5" } }, + "jsonld-context-parser": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/jsonld-context-parser/-/jsonld-context-parser-2.0.2.tgz", + "integrity": "sha512-IjQi26E+5ESS85MkcLsYo9gV93oJSOvQ/deHxKspaeHOmRiPyRRaGAk86DjuQc6c0hdp3OKGvDDsy3wi8znsqg==", + "requires": { + "@types/http-link-header": "^1.0.1", + "@types/isomorphic-fetch": "^0.0.35", + "@types/node": "^13.1.0", + "canonicalize": "^1.0.1", + "http-link-header": "^1.0.2", + "isomorphic-fetch": "^2.2.1", + "relative-to-absolute-iri": "^1.0.5" + }, + "dependencies": { + "@types/node": { + "version": "13.13.15", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.15.tgz", + "integrity": "sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw==" + } + } + }, + "jsonld-streaming-parser": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/jsonld-streaming-parser/-/jsonld-streaming-parser-2.0.2.tgz", + "integrity": "sha512-h4cK+PQMvOHd+UqgAFpKBmt5LWYoQMQLu9PP7YsRP7rnSJbU/EfJFcJFG6/sdtZslQ6dlNwOvfHbjQ9clzZPzg==", + "requires": { + "@rdfjs/data-model": "^1.1.2", + "@types/http-link-header": "^1.0.1", + "@types/rdf-js": "^3.0.0", + "canonicalize": "^1.0.1", + "http-link-header": "^1.0.2", + "jsonld-context-parser": "^2.0.1", + "jsonparse": "^1.3.1" + } + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -5460,6 +5778,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5870,6 +6193,14 @@ "lines-and-columns": "^1.1.6" } }, + "parse-link-header": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-1.0.1.tgz", + "integrity": "sha1-vt/g0hGK64S+deewJUGeyKYRQKc=", + "requires": { + "xtend": "~4.0.1" + } + }, "parse5": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", @@ -6008,6 +6339,11 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "promise-polyfill": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-1.1.6.tgz", + "integrity": "sha1-zQTv9G9clcOn0EVZHXm14+AfEtc=" + }, "prompts": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.3.2.tgz", @@ -6157,6 +6493,31 @@ "rdf-terms": "^1.4.0" } }, + "rdf-parse": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/rdf-parse/-/rdf-parse-1.5.0.tgz", + "integrity": "sha512-NtX08vnqQNzzdpxztPKzZ9U88udXu/mi1OvSnmKrqnao+B2T7VhU1Bn3A0c4VNmQdBL6V0P337BvzaEs9sqJtw==", + "requires": { + "@comunica/actor-http-native": "~1.16.0", + "@comunica/actor-rdf-parse-html": "~1.15.0", + "@comunica/actor-rdf-parse-html-rdfa": "~1.15.0", + "@comunica/actor-rdf-parse-html-script": "~1.16.1", + "@comunica/actor-rdf-parse-jsonld": "~1.16.0", + "@comunica/actor-rdf-parse-n3": "~1.15.0", + "@comunica/actor-rdf-parse-rdfxml": "~1.15.0", + "@comunica/actor-rdf-parse-xml-rdfa": "~1.15.0", + "@comunica/bus-http": "~1.16.0", + "@comunica/bus-init": "~1.15.0", + "@comunica/bus-rdf-parse": "~1.15.0", + "@comunica/bus-rdf-parse-html": "~1.15.0", + "@comunica/core": "~1.15.0", + "@comunica/mediator-combine-union": "~1.15.0", + "@comunica/mediator-number": "~1.15.0", + "@comunica/mediator-race": "~1.15.0", + "@types/rdf-js": "^3.0.0", + "stream-to-string": "^1.2.0" + } + }, "rdf-string": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/rdf-string/-/rdf-string-1.4.2.tgz", @@ -6174,6 +6535,27 @@ "lodash.uniqwith": "^4.5.0" } }, + "rdfa-streaming-parser": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/rdfa-streaming-parser/-/rdfa-streaming-parser-1.2.2.tgz", + "integrity": "sha512-OKNyZworn+wuHd9DsskyiBor85nQPAMzSR/xm6np1Pe09edj3yRGJQLsY62Ww1ELjZbdEFXougIShhR9VwU83A==", + "requires": { + "@rdfjs/data-model": "^1.1.1", + "@types/rdf-js": "^3.0.0", + "htmlparser2": "^4.0.0", + "relative-to-absolute-iri": "^1.0.2" + } + }, + "rdfxml-streaming-parser": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/rdfxml-streaming-parser/-/rdfxml-streaming-parser-1.3.6.tgz", + "integrity": "sha512-t9uqmCiPjmMFMXQ3Va2rc4ePElhze63EUmXVLA05s40NQEvphj8I8Kl1qODBwHnxocdoc1yVQRsC6hVJAPHvPQ==", + "requires": { + "@rdfjs/data-model": "^1.1.2", + "relative-to-absolute-iri": "^1.0.0", + "sax": "^1.2.4" + } + }, "react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", @@ -6226,6 +6608,11 @@ "util-deprecate": "~1.0.1" } }, + "readable-stream-node-to-web": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/readable-stream-node-to-web/-/readable-stream-node-to-web-1.0.1.tgz", + "integrity": "sha1-i3YU+qFGXr+g2pucpjA/onBzt88=" + }, "readdirp": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", @@ -6291,6 +6678,11 @@ "rc": "^1.2.8" } }, + "relative-to-absolute-iri": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/relative-to-absolute-iri/-/relative-to-absolute-iri-1.0.6.tgz", + "integrity": "sha512-Xw5/Zx6iWSCMJUXwXVOjySjH8Xli4hVFL9QQFvkl1qEmFBG94J+QUI9emnoctOCD3285f1jNV+QWV9eDYwIdfQ==" + }, "remove-trailing-separator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", @@ -6654,6 +7046,11 @@ } } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -7187,6 +7584,14 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stream-to-string": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/stream-to-string/-/stream-to-string-1.2.0.tgz", + "integrity": "sha512-8drZlFIKBHSMdX9GCWv8V9AAWnQcTqw0iAI6/GC7UJ0H0SwKeFKjOoZfGY1tOU00GGU7FYZQoJ/ZCUEoXhD7yQ==", + "requires": { + "promise-polyfill": "^1.1.6" + } + }, "streamify-array": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/streamify-array/-/streamify-array-1.0.1.tgz", @@ -7964,6 +8369,21 @@ "makeerror": "1.0.x" } }, + "web-streams-node": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/web-streams-node/-/web-streams-node-0.4.0.tgz", + "integrity": "sha512-u+PBQs8DFaBrN/bxCLFn21tO/ZP7EM3qA4FGzppoUCcZ5CaMbKOsN8uOp27ylVEsfrxcR2tsF6gWHI5M8bN73w==", + "requires": { + "is-stream": "^1.1.0", + "readable-stream-node-to-web": "^1.0.1", + "web-streams-ponyfill": "^1.4.1" + } + }, + "web-streams-ponyfill": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/web-streams-ponyfill/-/web-streams-ponyfill-1.4.2.tgz", + "integrity": "sha512-LCHW+fE2UBJ2vjhqJujqmoxh1ytEDEr0dPO3CabMdMDJPKmsaxzS90V1Ar6LtNE5VHLqxR4YMEj1i4lzMAccIA==" + }, "webidl-conversions": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", @@ -7979,6 +8399,11 @@ "iconv-lite": "0.4.24" } }, + "whatwg-fetch": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.0.tgz", + "integrity": "sha512-rsum2ulz2iuZH08mJkT0Yi6JnKhwdw4oeyMjokgxd+mmqYSd9cPpOQf01TIWgjxG/U4+QR+AwKq6lSbXVxkyoQ==" + }, "whatwg-mimetype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", @@ -8109,6 +8534,11 @@ "@babel/runtime-corejs3": "^7.8.3" } }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", diff --git a/package.json b/package.json index 38c7b4bba..66ad8e01a 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "express": "^4.17.1", "mime-types": "^2.1.27", "n3": "^1.4.0", + "rdf-parse": "^1.5.0", "rdf-terms": "^1.5.1", "sparqlalgebrajs": "^2.3.1", "uuid": "^8.3.0", diff --git a/src/storage/conversion/RdfToQuadConverter.ts b/src/storage/conversion/RdfToQuadConverter.ts new file mode 100644 index 000000000..73484691b --- /dev/null +++ b/src/storage/conversion/RdfToQuadConverter.ts @@ -0,0 +1,41 @@ +import { PassThrough } from 'stream'; +import rdfParser from 'rdf-parse'; +import { Representation } from '../../ldp/representation/Representation'; +import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata'; +import { CONTENT_TYPE_QUADS, DATA_TYPE_QUAD } from '../../util/ContentTypes'; +import { UnsupportedHttpError } from '../../util/errors/UnsupportedHttpError'; +import { checkRequest } from './ConversionUtil'; +import { RepresentationConverter, RepresentationConverterArgs } from './RepresentationConverter'; + +/** + * Converts most major RDF serializations to `internal/quads`. + */ +export class RdfToQuadConverter extends RepresentationConverter { + public async canHandle(input: RepresentationConverterArgs): Promise { + checkRequest(input, await rdfParser.getContentTypes(), [ CONTENT_TYPE_QUADS ]); + } + + public async handle(input: RepresentationConverterArgs): Promise { + return this.rdfToQuads(input.representation, input.identifier.path); + } + + private rdfToQuads(representation: Representation, baseIRI: string): Representation { + const metadata: RepresentationMetadata = { ...representation.metadata, contentType: CONTENT_TYPE_QUADS }; + + // Catch parsing errors and emit correct error + // Node 10 requires both writableObjectMode and readableObjectMode + const errorStream = new PassThrough({ writableObjectMode: true, readableObjectMode: true }); + const data = rdfParser.parse(representation.data, { + contentType: representation.metadata.contentType as string, + baseIRI, + }); + data.pipe(errorStream); + data.on('error', (error): boolean => errorStream.emit('error', new UnsupportedHttpError(error.message))); + + return { + dataType: DATA_TYPE_QUAD, + data: errorStream, + metadata, + }; + } +} diff --git a/test/unit/storage/conversion/RdfToQuadConverter.test.ts b/test/unit/storage/conversion/RdfToQuadConverter.test.ts new file mode 100644 index 000000000..10a0f8420 --- /dev/null +++ b/test/unit/storage/conversion/RdfToQuadConverter.test.ts @@ -0,0 +1,86 @@ +import { Readable } from 'stream'; +import { namedNode, triple } from '@rdfjs/data-model'; +import arrayifyStream from 'arrayify-stream'; +import streamifyArray from 'streamify-array'; +import { Representation } from '../../../../src/ldp/representation/Representation'; +import { RepresentationPreferences } from '../../../../src/ldp/representation/RepresentationPreferences'; +import { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier'; +import { RdfToQuadConverter } from '../../../../src/storage/conversion/RdfToQuadConverter'; +import { CONTENT_TYPE_QUADS, DATA_TYPE_QUAD } from '../../../../src/util/ContentTypes'; +import { UnsupportedHttpError } from '../../../../src/util/errors/UnsupportedHttpError'; + +describe('A RdfToQuadConverter.test.ts', (): void => { + const converter = new RdfToQuadConverter(); + const identifier: ResourceIdentifier = { path: 'path' }; + + it('can handle turtle to quad conversions.', async(): Promise => { + const representation = { metadata: { contentType: 'text/turtle' }} as Representation; + const preferences: RepresentationPreferences = { type: [{ value: CONTENT_TYPE_QUADS, weight: 1 }]}; + await expect(converter.canHandle({ identifier, representation, preferences })).resolves.toBeUndefined(); + }); + + it('can handle JSON-LD to quad conversions.', async(): Promise => { + const representation = { metadata: { contentType: 'application/ld+json' }} as Representation; + const preferences: RepresentationPreferences = { type: [{ value: CONTENT_TYPE_QUADS, weight: 1 }]}; + await expect(converter.canHandle({ identifier, representation, preferences })).resolves.toBeUndefined(); + }); + + it('converts turtle to quads.', async(): Promise => { + const representation = { + data: streamifyArray([ ' .' ]), + metadata: { contentType: 'text/turtle' }, + } as Representation; + const preferences: RepresentationPreferences = { type: [{ value: CONTENT_TYPE_QUADS, weight: 1 }]}; + const result = await converter.handle({ identifier, representation, preferences }); + expect(result).toEqual({ + data: expect.any(Readable), + dataType: DATA_TYPE_QUAD, + metadata: { + contentType: CONTENT_TYPE_QUADS, + }, + }); + await expect(arrayifyStream(result.data)).resolves.toEqualRdfQuadArray([ triple( + namedNode('http://test.com/s'), + namedNode('http://test.com/p'), + namedNode('http://test.com/o'), + ) ]); + }); + + it('converts JSON-LD to quads.', async(): Promise => { + const representation = { + data: streamifyArray([ '{"@id": "http://test.com/s", "http://test.com/p": { "@id": "http://test.com/o" }}' ]), + metadata: { contentType: 'application/ld+json' }, + } as Representation; + const preferences: RepresentationPreferences = { type: [{ value: CONTENT_TYPE_QUADS, weight: 1 }]}; + const result = await converter.handle({ identifier, representation, preferences }); + expect(result).toEqual({ + data: expect.any(Readable), + dataType: DATA_TYPE_QUAD, + metadata: { + contentType: CONTENT_TYPE_QUADS, + }, + }); + await expect(arrayifyStream(result.data)).resolves.toEqualRdfQuadArray([ triple( + namedNode('http://test.com/s'), + namedNode('http://test.com/p'), + namedNode('http://test.com/o'), + ) ]); + }); + + it('throws an UnsupportedHttpError on invalid triple data.', async(): Promise => { + const representation = { + data: streamifyArray([ '