From adfb8724be28f52d9f72375b57eb88712fa9fb2e Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Thu, 9 Jul 2020 18:05:42 +0530 Subject: [PATCH 1/7] Implement http based callback --- bin/callback.js | 84 +++++++++++++++++++++++++++++++++++++++++++++++ bin/utils.js | 15 +++++++++ package-lock.json | 5 +++ package.json | 1 + 4 files changed, 105 insertions(+) create mode 100644 bin/callback.js diff --git a/bin/callback.js b/bin/callback.js new file mode 100644 index 0000000..f9acc27 --- /dev/null +++ b/bin/callback.js @@ -0,0 +1,84 @@ +const http = require('http') + +const CALLBACK_URL = process.env.CALLBACK_URL ? new URL(process.env.CALLBACK_URL) : null +const CALLBACK_TIMEOUT = process.env.CALLBACK_TIMEOUT || 5000 +const CALLBACK_OBJECTS = process.env.CALLBACK_OBJECTS ? JSON.parse(process.env.CALLBACK_OBJECTS) : {} + +/** + * @return {Boolean} + */ +exports.isCallbackSet = () => { + if (CALLBACK_URL) { + return true + } + return false +} + +/** + * @param {Uint8Array} update + * @param {any} origin + * @param {WSSharedDoc} doc + */ +exports.callbackHandler = (update, origin, doc) => { + const room = doc.name + const dataToSend = { + room: room, + data: {} + } + const sharedObjectList = Object.keys(CALLBACK_OBJECTS) + sharedObjectList.forEach(sharedObjectName => { + const sharedObjectType = CALLBACK_OBJECTS[sharedObjectName] + dataToSend.data[sharedObjectName] = { + type: sharedObjectType, + content: getContent(sharedObjectName, sharedObjectType, doc) + } + }) + callbackRequest(CALLBACK_URL, CALLBACK_TIMEOUT, dataToSend) +} + +/** + * @param {URL} url + * @param {number} timeout + * @param {Object} data + */ +const callbackRequest = (url, timeout, data) => { + data = JSON.stringify(data) + const options = { + hostname: url.hostname, + port: url.port, + path: url.pathname, + timeout: timeout, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + } + var req = http.request(options) + req.on('timeout', () => { + console.warn('Callback request timed out.') + req.abort() + }) + req.on('error', (e) => { + console.err('Callback request error.', e) + req.abort() + }) + req.write(data) + req.end() +} + +/** + * @param {string} objName + * @param {string} objType + * @param {WSSharedDoc} doc + */ +const getContent = (objName, objType, doc) => { + switch (objType) { + case 'Array': return doc.getArray(objName) + case 'Map': return doc.getMap(objName) + case 'Text': return doc.getText(objName) + case 'XmlFragment': return doc.getXmlFragment(objName) + case 'XmlElement': return doc.getXmlElement(objName) + default : return {} + } +} diff --git a/bin/utils.js b/bin/utils.js index 3088775..5e2cb08 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -7,6 +7,14 @@ const decoding = require('lib0/dist/decoding.cjs') const mutex = require('lib0/dist/mutex.cjs') const map = require('lib0/dist/map.cjs') +const debounce = require('lodash.debounce') + +const callbackHandler = require('./callback.js').callbackHandler +const isCallbackSet = require('./callback.js').isCallbackSet + +const CALLBACK_DEBOUNCE_WAIT = process.env.CALLBACK_DEBOUNCE_WAIT || 2000 +const CALLBACK_DEBOUNCE_MAXWAIT = process.env.CALLBACK_DEBOUNCE_MAXWAIT || 10000 + const wsReadyStateConnecting = 0 const wsReadyStateOpen = 1 const wsReadyStateClosing = 2 // eslint-disable-line @@ -110,6 +118,13 @@ class WSSharedDoc extends Y.Doc { } this.awareness.on('update', awarenessChangeHandler) this.on('update', updateHandler) + if (isCallbackSet()) { + this.on('update', debounce( + callbackHandler, + CALLBACK_DEBOUNCE_WAIT, + { maxWait: CALLBACK_DEBOUNCE_MAXWAIT } + )) + } } } diff --git a/package-lock.json b/package-lock.json index 5f8b856..9533b0b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1192,6 +1192,11 @@ "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", diff --git a/package.json b/package.json index 412e979..688978e 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "dependencies": { "lib0": "^0.2.31", "y-leveldb": "^0.1.0", + "lodash.debounce": "^4.0.8", "y-protocols": "^1.0.0" }, "devDependencies": { From 968e71e038f436948cbd817d9c19f63f42934ed7 Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Sun, 12 Jul 2020 21:11:42 +0530 Subject: [PATCH 2/7] Update ReadMe for HTTP Callbacks --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 0e61288..bc12eea 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,22 @@ See [LevelDB Persistence](https://github.com/yjs/y-leveldb) for more info. PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/y-websocket/bin/server.js ``` +**Websocket Server with HTTP callback** + +Send a debounced callback to an HTTP server (POST) on document update. + +Can take the following ENV variables: +- CALLBACK_URL : Callback server URL. +- CALLBACK_DEBOUNCE_WAIT : Debounce time between callbacks. Defaults to 2 seconds. +- CALLBACK_DEBOUNCE_MAXWAIT : Maximum time to wait before callback. Defaults to 10 seconds. +- CALLBACK_TIMEOUT : Timeout for the HTTP call. Defaults to 5 seconds. +- CALLBACK_OBJECTS : JSON of shared objects to get data. ('{"SHARED_OBJECT_NAME":"SHARED_OBJECT_TYPE}') + +```sh +CALLBACK_URL=http://localhost:3000/ CALLBACK_OBJECTS='{"prosemirror":"XmlFragment"}' node ./node_modules/y-websocket/bin/server.js +``` +This would send a callback to localhost:3000 every 2 seconds (default DEBOUNCE_WAIT) with the data of an XmlFragment named Prosemirror in the body. + ### Scaling These are mere suggestions how you could scale your server environment. From fe0c51375ac3f0105b7df3e6bb16a2d5bd54259d Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Thu, 30 Jul 2020 02:30:44 +0530 Subject: [PATCH 3/7] Change isCallbackSet from function call to Boolean constant --- bin/callback.js | 10 +--------- bin/utils.js | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/bin/callback.js b/bin/callback.js index f9acc27..1bc0444 100644 --- a/bin/callback.js +++ b/bin/callback.js @@ -4,15 +4,7 @@ const CALLBACK_URL = process.env.CALLBACK_URL ? new URL(process.env.CALLBACK_URL const CALLBACK_TIMEOUT = process.env.CALLBACK_TIMEOUT || 5000 const CALLBACK_OBJECTS = process.env.CALLBACK_OBJECTS ? JSON.parse(process.env.CALLBACK_OBJECTS) : {} -/** - * @return {Boolean} - */ -exports.isCallbackSet = () => { - if (CALLBACK_URL) { - return true - } - return false -} +exports.isCallbackSet = !!CALLBACK_URL /** * @param {Uint8Array} update diff --git a/bin/utils.js b/bin/utils.js index 5e2cb08..1138961 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -118,7 +118,7 @@ class WSSharedDoc extends Y.Doc { } this.awareness.on('update', awarenessChangeHandler) this.on('update', updateHandler) - if (isCallbackSet()) { + if (isCallbackSet) { this.on('update', debounce( callbackHandler, CALLBACK_DEBOUNCE_WAIT, From 7d8d9a4a45648044a061ad4abc9d6f8a771728ba Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Thu, 30 Jul 2020 02:43:36 +0530 Subject: [PATCH 4/7] Serialize Yjs type before callback --- bin/callback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/callback.js b/bin/callback.js index 1bc0444..2390b74 100644 --- a/bin/callback.js +++ b/bin/callback.js @@ -22,7 +22,7 @@ exports.callbackHandler = (update, origin, doc) => { const sharedObjectType = CALLBACK_OBJECTS[sharedObjectName] dataToSend.data[sharedObjectName] = { type: sharedObjectType, - content: getContent(sharedObjectName, sharedObjectType, doc) + content: getContent(sharedObjectName, sharedObjectType, doc).toJSON() } }) callbackRequest(CALLBACK_URL, CALLBACK_TIMEOUT, dataToSend) From a7d805cf43d09c991c2b39d36df5607890a3de06 Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Thu, 30 Jul 2020 02:44:33 +0530 Subject: [PATCH 5/7] Fix callback request error logging --- bin/callback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/callback.js b/bin/callback.js index 2390b74..d53d7cc 100644 --- a/bin/callback.js +++ b/bin/callback.js @@ -52,7 +52,7 @@ const callbackRequest = (url, timeout, data) => { req.abort() }) req.on('error', (e) => { - console.err('Callback request error.', e) + console.error('Callback request error.', e) req.abort() }) req.write(data) From f61b36424843ae44b44dd4253b17ad42e3e8518c Mon Sep 17 00:00:00 2001 From: Mansehej Singh Date: Thu, 30 Jul 2020 03:02:43 +0530 Subject: [PATCH 6/7] Change run script in ReadMe callback example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc12eea..09d4aba 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Can take the following ENV variables: - CALLBACK_OBJECTS : JSON of shared objects to get data. ('{"SHARED_OBJECT_NAME":"SHARED_OBJECT_TYPE}') ```sh -CALLBACK_URL=http://localhost:3000/ CALLBACK_OBJECTS='{"prosemirror":"XmlFragment"}' node ./node_modules/y-websocket/bin/server.js +CALLBACK_URL=http://localhost:3000/ CALLBACK_OBJECTS='{"prosemirror":"XmlFragment"}' npm start ``` This would send a callback to localhost:3000 every 2 seconds (default DEBOUNCE_WAIT) with the data of an XmlFragment named Prosemirror in the body. From f7e5d4cb76c535bb1457f7ea2f996d8df981fb63 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 8 Aug 2020 18:45:16 +0200 Subject: [PATCH 7/7] personal style update --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 09d4aba..e95cd57 100644 --- a/README.md +++ b/README.md @@ -42,19 +42,20 @@ PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/y-websocket/bin/server.js **Websocket Server with HTTP callback** -Send a debounced callback to an HTTP server (POST) on document update. +Send a debounced callback to an HTTP server (`POST`) on document update. Can take the following ENV variables: -- CALLBACK_URL : Callback server URL. -- CALLBACK_DEBOUNCE_WAIT : Debounce time between callbacks. Defaults to 2 seconds. -- CALLBACK_DEBOUNCE_MAXWAIT : Maximum time to wait before callback. Defaults to 10 seconds. -- CALLBACK_TIMEOUT : Timeout for the HTTP call. Defaults to 5 seconds. -- CALLBACK_OBJECTS : JSON of shared objects to get data. ('{"SHARED_OBJECT_NAME":"SHARED_OBJECT_TYPE}') + +* `CALLBACK_URL` : Callback server URL +* `CALLBACK_DEBOUNCE_WAIT` : Debounce time between callbacks (in ms). Defaults to 2000 ms +* `CALLBACK_DEBOUNCE_MAXWAIT` : Maximum time to wait before callback. Defaults to 10 seconds +* `CALLBACK_TIMEOUT` : Timeout for the HTTP call. Defaults to 5 seconds +* `CALLBACK_OBJECTS` : JSON of shared objects to get data (`'{"SHARED_OBJECT_NAME":"SHARED_OBJECT_TYPE}'`) ```sh CALLBACK_URL=http://localhost:3000/ CALLBACK_OBJECTS='{"prosemirror":"XmlFragment"}' npm start ``` -This would send a callback to localhost:3000 every 2 seconds (default DEBOUNCE_WAIT) with the data of an XmlFragment named Prosemirror in the body. +This sends a debounced callback to `localhost:3000` 2 seconds after receiving an update (default `DEBOUNCE_WAIT`) with the data of an XmlFragment named `"prosemirror"` in the body. ### Scaling