From f6943f33af784a7027de63e7c1506801b8c190d5 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 10 Jul 2020 02:39:44 +0200 Subject: [PATCH] Fix YPERSISTENCE -- allowing y-leveldb persistence --- .gitignore | 1 + README.md | 2 +- bin/utils.js | 17 +++- package-lock.json | 237 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 3 +- 5 files changed, 252 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index de4d1f0..5f6c423 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ dist node_modules +tmp \ No newline at end of file diff --git a/README.md b/README.md index 78c5d02..0e61288 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Since npm symlinks the `y-websocket-server` executable from your local `./node_m Persist document updates in a LevelDB database. -See [LevelDB Persistence](#LevelDB Persistence) for more info. +See [LevelDB Persistence](https://github.com/yjs/y-leveldb) for more info. ```sh PORT=1234 YPERSISTENCE=./dbDir node ./node_modules/y-websocket/bin/server.js diff --git a/bin/utils.js b/bin/utils.js index 0ad5b40..3088775 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -20,9 +20,22 @@ const persistenceDir = process.env.YPERSISTENCE */ let persistence = null if (typeof persistenceDir === 'string') { + console.info('Persisting documents to "' + persistenceDir + '"') // @ts-ignore - const LevelDbPersistence = require('y-leveldb').LevelDbPersistence - persistence = new LevelDbPersistence(persistenceDir) + const LeveldbPersistence = require('y-leveldb').LeveldbPersistence + const ldb = new LeveldbPersistence(persistenceDir) + persistence = { + bindState: async (docName, ydoc) => { + const persistedYdoc = await ldb.getYDoc(docName) + const newUpdates = Y.encodeStateAsUpdate(ydoc) + ldb.storeUpdate(docName, newUpdates) + Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc)) + ydoc.on('update', update => { + ldb.storeUpdate(docName, update) + }) + }, + writeState: async (docName, ydoc) => {} + } } /** diff --git a/package-lock.json b/package-lock.json index 7a952de..d9d9ff6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,18 @@ "integrity": "sha512-3ySmiBYJPqgjiHA7oEaIo2Rzz0HrOZ7yrNO5HWyaE5q0lQ3BppDZ3N53Miz8bw2I7gh1/zir2MGVZBvpb1zq9g==", "dev": true }, + "abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "requires": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + } + }, "acorn": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.2.0.tgz", @@ -130,6 +142,11 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -140,6 +157,15 @@ "concat-map": "0.0.1" } }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "caller-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz", @@ -274,6 +300,15 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, + "deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "requires": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + } + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -314,6 +349,25 @@ "esutils": "^2.0.2" } }, + "encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "requires": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + } + }, + "errno": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "requires": { + "prr": "~1.0.1" + } + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -810,12 +864,22 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -835,8 +899,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "inquirer": { "version": "5.2.0", @@ -973,6 +1036,116 @@ "array-includes": "^3.0.3" } }, + "level": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", + "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "requires": { + "level-js": "^5.0.0", + "level-packager": "^5.1.0", + "leveldown": "^5.4.0" + } + }, + "level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "requires": { + "buffer": "^5.6.0" + } + }, + "level-concat-iterator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", + "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" + }, + "level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "requires": { + "errno": "~0.1.1" + } + }, + "level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + }, + "dependencies": { + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } + }, + "level-js": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", + "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "requires": { + "abstract-leveldown": "~6.2.3", + "buffer": "^5.5.0", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + } + }, + "level-packager": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", + "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "requires": { + "encoding-down": "^6.3.0", + "levelup": "^4.3.2" + } + }, + "level-supports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", + "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "requires": { + "xtend": "^4.0.2" + }, + "dependencies": { + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } + }, + "leveldown": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", + "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "requires": { + "abstract-leveldown": "~6.2.1", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.1.0" + } + }, + "levelup": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", + "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "requires": { + "deferred-leveldown": "~5.3.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~4.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -1028,6 +1201,11 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" + }, "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -1070,6 +1248,11 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, + "napi-macros": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -1082,6 +1265,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-gyp-build": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", + "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -1311,6 +1499,11 @@ "react-is": "^16.8.1" } }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -1344,6 +1537,16 @@ "read-pkg": "^2.0.0" } }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", @@ -1435,6 +1638,11 @@ "symbol-observable": "1.0.1" } }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1562,6 +1770,14 @@ "strip-ansi": "^4.0.0" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -1668,6 +1884,11 @@ "punycode": "^2.1.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -1720,8 +1941,16 @@ "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", - "dev": true + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, + "y-leveldb": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.0.tgz", + "integrity": "sha512-sMuitVrsAUNh+0b66I42nAuW3lCmez171uP4k0ePcTAJ+c+Iw9w4Yq3wwiyrDMFXBEyQSjSF86Inc23wEvWnxw==", + "requires": { + "level": "^6.0.1", + "lib0": "^0.2.31" + } }, "y-protocols": { "version": "1.0.0", diff --git a/package.json b/package.json index e222488..6c15871 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "dist": "rm -rf dist && rollup -c && tsc", "lint": "standard && tsc", "test": "npm run lint", - "preversion": "npm run lint && npm run dist" + "preversion": "npm run lint && npm run dist && test -e dist/src/y-websocket.d.ts && test -e dist/y-websocket.cjs" }, "bin": { "y-websocket-server": "./bin/server.js" @@ -46,6 +46,7 @@ }, "dependencies": { "lib0": "^0.2.31", + "y-leveldb": "^0.1.0", "y-protocols": "^1.0.0" }, "devDependencies": {