From e216cec51da5e9cc5bae256a1361c517d2c28fda Mon Sep 17 00:00:00 2001 From: Gabriel Schulhof Date: Wed, 18 Oct 2017 18:40:13 +0300 Subject: [PATCH] Create binary prebuilds * Introduce infrastructure for creating prebuilds and uploading them to a github repo. * Use prebuilds on all platforms except Windows and except Node.js >= 8.0.0 and < 8.6.0. * Update documentation to reflect requirement to load native addon `loadModule()` provided by index.js instead of `require()`. * Update tests to use `loadModule()`. --- .travis.yml | 1 + PREBUILD.md | 19 ++++++++++++++ README.md | 18 ++++++++++--- binding.gyp | 37 ++++++++++++++++++++++++++ index.js | 42 +++++++++++++++++++++++++----- loader/main.c | 27 +++++++++++++++++++ package.json | 18 ++++++++++--- scripts/postinstall.js | 53 ++++++++++++++++++++++++++++++++++++++ scripts/upload-binaries.js | 49 +++++++++++++++++++++++++++++++++++ src/node_api.gyp | 5 +++- test/arraybuffer.js | 5 ++-- test/asyncworker.js | 5 ++-- test/buffer.js | 5 ++-- test/error.js | 13 +++++----- test/external.js | 5 ++-- test/function.js | 5 ++-- test/load-bindings.js | 9 +++++++ test/name.js | 5 ++-- test/object.js | 5 ++-- test/objectwrap.js | 5 ++-- test/typedarray.js | 5 ++-- 21 files changed, 287 insertions(+), 49 deletions(-) create mode 100644 PREBUILD.md create mode 100644 binding.gyp create mode 100644 loader/main.c create mode 100644 scripts/postinstall.js create mode 100644 scripts/upload-binaries.js create mode 100644 test/load-bindings.js diff --git a/.travis.yml b/.travis.yml index c37486fff..7411f5988 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,7 @@ before_install: - npm --version install: - npm install $NPMOPT + - npm run-script preparetest $NPMOPT script: # Travis CI sets NVM_NODEJS_ORG_MIRROR, but it makes node-gyp fail to download headers for nightly builds. - unset NVM_NODEJS_ORG_MIRROR diff --git a/PREBUILD.md b/PREBUILD.md new file mode 100644 index 000000000..00df212c0 --- /dev/null +++ b/PREBUILD.md @@ -0,0 +1,19 @@ +## Creating a prebuild for a certain platform/arch + +1. Set up an ssh host which must be named `napi-binary-upload` for committing to +the repo napi-binary-upload/upload. This is done in case the ssh key for +committing to the napi-binary-upload/upload repo must be different than the one +for committing to other github repositories. + ``` + host napi-binary-upload + Hostname github.com + User git + IdentityFile ~/.ssh/napi-binary-upload-ssh-key + ``` + + The ssh secret key file is optional and can be at any path and it may have + any name. + +2. `git clean -x -d -f -f` +3. `npm install` +4. `npm run-script binaries` diff --git a/README.md b/README.md index 259f5939d..55e1c1326 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ To use N-API in a native module: 2. Reference this package's include directory and gyp file in `binding.gyp`: ```gyp - 'include_dirs': [" 8 || (versionArray[0] == 8 && versionArray[1] > 5)); - -// So far it looks like even version 9 will need the flag. We need to adjust -// this for the version where the flag is dropped whenever that version lands. -var needsFlag = (versionArray[0] >= 8); +var needsFlag = (!isNodeApiBuiltin && versionArray[0] == 8); var include = path.join(__dirname, 'src'); var gyp = path.join(__dirname, 'src', 'node_api.gyp'); -if (isNodeApiBuiltin) { +if (isNodeApiBuiltin || (process.platform !== 'win32' && !needsFlag)) { gyp += ':nothing'; } else { gyp += ':node-api'; include = path.join(__dirname, 'external-napi'); } +function loadModule(filename) { + var result; + if (!isNodeApiBuiltin && process.platform !== 'win32') { + + // On non-Windows node versions wheren N-API is external we tell + // n-api-loader what implementation and what module to load via environment + // variables. + process.env.NAPI_MODULE_FILE_NAME = filename; + process.env.NAPI_IMPLEM_FILE_NAME = + path.join(__dirname, 'implems', + 'n-api-implem-v' + process.versions.modules + '.node'); + + // We try loading n-api-loader from both Release and Debug. + try { + result = require('./build/Release/n-api-loader.node'); + } catch (anError) { + try { + result = require('./build/Debug/n-api-loader.node'); + } catch (anError) {} + } + + // We remove the environment variables we added. + delete process.env.NAPI_MODULE_FILE_NAME; + delete process.env.NAPI_IMPLEM_FILE_NAME; + } + + // Finally, if loading the module via n-api-loader failed or we haven't even + // tried because we're on win32 or N-API is built-in, we load the module the + // conventional way. + return result || module.parent.require(filename); +} + module.exports = { include: [ '"' + include + '"', '"' + __dirname + '"' ].join(' '), gyp: gyp, isNodeApiBuiltin: isNodeApiBuiltin, - needsFlag: needsFlag + needsFlag: needsFlag, + loadModule: loadModule }; diff --git a/loader/main.c b/loader/main.c new file mode 100644 index 000000000..39ccd955c --- /dev/null +++ b/loader/main.c @@ -0,0 +1,27 @@ +#include +#include +#include + +#define NAPI_MODULE_FILE_NAME "NAPI_MODULE_FILE_NAME" +#define NAPI_IMPLEM_FILE_NAME "NAPI_IMPLEM_FILE_NAME" + +int load_one(char* filename, int flags) { + void* handle = dlopen(filename, flags); + if (handle == NULL) { + fprintf(stderr, "Loading %s: %s\n", filename, dlerror()); + return -1; + } + return 0; +} + +void load_napi_module(void) __attribute__((constructor)); +void load_napi_module(void) { + char* module_file_name = getenv(NAPI_MODULE_FILE_NAME); + char* implem_file_name = getenv(NAPI_IMPLEM_FILE_NAME); + + if (module_file_name != NULL && implem_file_name != NULL) { + if (load_one(implem_file_name, RTLD_LAZY | RTLD_GLOBAL) == 0) { + load_one(module_file_name, RTLD_LAZY); + } + } +} diff --git a/package.json b/package.json index f2883aceb..9cc78d235 100644 --- a/package.json +++ b/package.json @@ -13,11 +13,18 @@ "Sampson Gao (https://github.com/sampsongao)", "Taylor Woll (https://github.com/boingoing)" ], - "dependencies": {}, + "dependencies": { + "tar-stream": "^1.5.4" + }, "description": "Node.js API (N-API)", "devDependencies": { - "safe-buffer": "^5.1.1" + "bindings": "^1.3.0", + "lodash.assignin": "^4.2.0", + "prebuild": "^6.2.2", + "safe-buffer": "^5.1.1", + "shelljs": "^0.7.8" }, + "binariesPage": "https://raw.githubusercontent.com/napi-binary-upload/upload/master/", "directories": {}, "homepage": "https://github.com/nodejs/node-addon-api", "license": "MIT", @@ -30,9 +37,12 @@ "url": "git://github.com/nodejs/node-addon-api.git" }, "scripts": { - "pretest": "node-gyp rebuild -C test", + "preparetest": "node-gyp rebuild -C test", "test": "node test", - "doc": "doxygen doc/Doxyfile" + "doc": "doxygen doc/Doxyfile", + "postinstall": "node scripts/postinstall.js", + "binaries": "RUNNING_FROM_PREBUILD=true prebuild -t 4.0.0 -t 6.0.0 -t 8.0.0", + "postbinaries": "node scripts/upload-binaries.js" }, "version": "1.0.0" } diff --git a/scripts/postinstall.js b/scripts/postinstall.js new file mode 100644 index 000000000..66c0a44de --- /dev/null +++ b/scripts/postinstall.js @@ -0,0 +1,53 @@ +// Grab the prebuilds relevant to this platform and arch and store them in +// ./implems. + +var https = require('https'); +var pkg = require('../package.json'); +var homepage = require('url').parse(pkg.binariesPage); +var path = require('path'); +var tar = require('tar-stream'); +var zlib = require('zlib'); +var version = 'v' + pkg.version; +var fs = require('fs'); +var implemDir = path.resolve(path.join(__dirname, '..', 'implems')); + +// Request a prebuild for the given module version. +function requestOne(moduleVersion) { + var options = { + host: homepage.host, + + // We always join the paths with a "/" because this is for a URL + path: [ homepage.pathname, [ + pkg.name, + version, + 'node', + moduleVersion, + process.platform, + process.arch + ].join('-') + '.tar.gz'].join( '/' ) + }; + var extract = tar.extract(); + extract.on('entry', function(header, stream, next) { + // The prebuild utility creates tarballs containing exactly one file, so we + // expect that there will only be one "entry" event per request and that we + // will thus not overwrite multiple times the file into which the + // WriteStream empties. + stream.pipe(fs.createWriteStream( + path.join(implemDir, 'n-api-implem-' + moduleVersion + '.node'))); + }); + https.request(options, function(response) { + if (response.statusCode === 200) { + response.pipe(zlib.createGunzip()).pipe(extract); + } + }).end(); +} + +// Download all the prebuilds. +fs.mkdir(implemDir, function(error) { + if (error && error.code != 'EEXIST') { + throw error; + } + requestOne("v46"); + requestOne("v48"); + requestOne("v57"); +}); diff --git a/scripts/upload-binaries.js b/scripts/upload-binaries.js new file mode 100644 index 000000000..8324c7d84 --- /dev/null +++ b/scripts/upload-binaries.js @@ -0,0 +1,49 @@ +const fs = require('fs'); +const path = require('path'); +const shelljs = require('shelljs'); +const spawnSync = require( "child_process" ).spawnSync; +const pkg = require('../package.json'); +const assignIn = require('lodash.assignin'); + +const repoPath = path.resolve(path.join(__dirname, '..', 'napi-binary-upload')); +const prebuildsPath = path.resolve(path.join(__dirname, '..', 'prebuilds')); + +function run( command, args, options ) { + options = options || {}; + const result = spawnSync( command, args, assignIn( {}, options, { + stdio: [ process.stdin, process.stdout, process.stderr ], + shell: true + } ) ); + + if ( result.status !== 0 ) { + process.exit( result.status || 1 ); + } +} + +run('git', [ + 'clone', + 'napi-binary-upload:napi-binary-upload/upload', + repoPath +]); + +shelljs.cp.apply(shelljs, [ + '-f', + fs.readdirSync(prebuildsPath).map((item) => {return path.join(prebuildsPath, item)}), + repoPath +]); + +run('git', ['add'] + .concat(fs + .readdirSync(repoPath) + .filter((item) => {return item !== '.git';})), +{cwd: repoPath}); + +run('git', [ + 'commit', + '-m', + '"' + [pkg.version, process.platform, process.arch].join(' ') + '"' +], {cwd: repoPath}); + +run('git', ['push', 'origin', 'master'], {cwd: repoPath}); + +shelljs.rm('-rf', repoPath); diff --git a/src/node_api.gyp b/src/node_api.gyp index fa34d8308..87dfa345c 100644 --- a/src/node_api.gyp +++ b/src/node_api.gyp @@ -13,7 +13,10 @@ 'defines': [ 'EXTERNAL_NAPI', ], - 'cflags_cc': ['-fvisibility=hidden'] + 'cflags_cc': ['-fvisibility=hidden'], + 'xcode_settings': { + 'OTHER_CFLAGS': ['-fvisibility=hidden'] + } } ] } diff --git a/test/arraybuffer.js b/test/arraybuffer.js index d284fe84a..bb49e6f9a 100644 --- a/test/arraybuffer.js +++ b/test/arraybuffer.js @@ -1,10 +1,9 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); const testUtil = require('./testUtil'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { testUtil.runGCTests([ diff --git a/test/asyncworker.js b/test/asyncworker.js index 92d2458aa..45fb34c84 100644 --- a/test/asyncworker.js +++ b/test/asyncworker.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { binding.asyncworker.doWork(true, function (e) { diff --git a/test/buffer.js b/test/buffer.js index 0c6e64895..7c67c7b33 100644 --- a/test/buffer.js +++ b/test/buffer.js @@ -1,11 +1,10 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); const testUtil = require('./testUtil'); const safeBuffer = require('safe-buffer'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { testUtil.runGCTests([ diff --git a/test/error.js b/test/error.js index f41874d55..7bfb080be 100644 --- a/test/error.js +++ b/test/error.js @@ -1,18 +1,17 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); if (process.argv[2] === 'fatal') { - const binding = require(process.argv[3]); + const binding = require('./load-bindings')(process.argv[3] === 'true' ? + 'binding' : 'binding_noexcept'); binding.error.throwFatalError(); return; } -test(`./build/${buildType}/binding.node`); -test(`./build/${buildType}/binding_noexcept.node`); +test(require('./load-bindings')('binding'), true); +test(require('./load-bindings')('binding_noexcept'), false); -function test(bindingPath) { - const binding = require(bindingPath); +function test(binding, withExceptions) { assert.throws(() => binding.error.throwApiError('test'), function(err) { return err instanceof Error && err.message.includes('Invalid'); @@ -66,7 +65,7 @@ function test(bindingPath) { }); const p = require('./napi_child').spawnSync( - process.execPath, [ __filename, 'fatal', bindingPath ]); + process.execPath, [ __filename, 'fatal', "" + withExceptions ]); assert.ifError(p.error); assert.ok(p.stderr.toString().includes( 'FATAL ERROR: Error::ThrowFatalError This is a fatal error')); diff --git a/test/external.js b/test/external.js index e2067006a..c44f9a378 100644 --- a/test/external.js +++ b/test/external.js @@ -1,10 +1,9 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); const testUtil = require('./testUtil'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { testUtil.runGCTests([ diff --git a/test/function.js b/test/function.js index 0d75ffca9..ed465d49a 100644 --- a/test/function.js +++ b/test/function.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { let obj = {}; diff --git a/test/load-bindings.js b/test/load-bindings.js new file mode 100644 index 000000000..5e46314d0 --- /dev/null +++ b/test/load-bindings.js @@ -0,0 +1,9 @@ +'use strict'; +module.exports = function loadBindings(name) { + return require('../index') + .loadModule(require('bindings')({ + bindings: name, + path: true, + module_root: __dirname + })); +}; diff --git a/test/name.js b/test/name.js index 4e9659312..695ce5e9d 100644 --- a/test/name.js +++ b/test/name.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { const expected = '123456789'; diff --git a/test/object.js b/test/object.js index 470d6fd0f..af2e9f057 100644 --- a/test/object.js +++ b/test/object.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { function assertPropertyIs(obj, key, attribute) { diff --git a/test/objectwrap.js b/test/objectwrap.js index 8aa5618f3..0860a751f 100644 --- a/test/objectwrap.js +++ b/test/objectwrap.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { var Test = binding.objectwrap.Test; diff --git a/test/typedarray.js b/test/typedarray.js index 9aa880c16..a817a80b1 100644 --- a/test/typedarray.js +++ b/test/typedarray.js @@ -1,9 +1,8 @@ 'use strict'; -const buildType = process.config.target_defaults.default_configuration; const assert = require('assert'); -test(require(`./build/${buildType}/binding.node`)); -test(require(`./build/${buildType}/binding_noexcept.node`)); +test(require('./load-bindings')('binding')); +test(require('./load-bindings')('binding_noexcept')); function test(binding) { const testData = [