From d3d54aa8ef6e4df04507f3d3a1b891c863eb497b Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 7 Aug 2018 13:09:04 +0200 Subject: [PATCH 01/17] build: extract common parts from addon .buildstamp This commit extracts common parts from the targets test/addons/.buildstamp, and test/addons-napi/.buildstamp to reduce some duplication. PR-URL: https://github.com/nodejs/node/pull/22171 Reviewed-By: Jon Moss --- Makefile | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index fc4a094656a4..f1242bbb30fc 100644 --- a/Makefile +++ b/Makefile @@ -333,19 +333,26 @@ ADDONS_BINDING_SOURCES := \ $(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \ $(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h)) +ADDONS_PREREQS := config.gypi \ + deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ + deps/uv/include/*.h deps/v8/include/*.h \ + src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h + +define run_build_addons +env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ + npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ + "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \ + $1 +touch $2 +endef + # Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale. # Depends on node-gyp package.json so that build-addons is (re)executed when # node-gyp is updated as part of an npm update. -test/addons/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ +test/addons/.buildstamp: $(ADDONS_PREREQS) \ $(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ test/addons/.docbuildstamp - env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ - npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ - "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" "$$PWD/test/addons" - touch $@ + @$(call run_build_addons,"$$PWD/test/addons",$@) .PHONY: build-addons # .buildstamp needs $(NODE_EXE) but cannot depend on it @@ -366,17 +373,10 @@ ADDONS_NAPI_BINDING_SOURCES := \ $(filter-out test/addons-napi/??_*/*.h, $(wildcard test/addons-napi/*/*.h)) # Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale. -test/addons-napi/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ +test/addons-napi/.buildstamp: $(ADDONS_PREREQS) \ $(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ src/node_api.h src/node_api_types.h - env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ - npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ - "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \ - "$$PWD/test/addons-napi" - touch $@ + @$(call run_build_addons,"$$PWD/test/addons-napi",$@) .PHONY: build-addons-napi # .buildstamp needs $(NODE_EXE) but cannot depend on it From 77da6d9e8c9496269d19dab00ea6f7f0b37e2839 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Mon, 6 Aug 2018 09:26:36 +0200 Subject: [PATCH 02/17] build: add crypto check to build targets Currently when configured without-ssl the build will fail when trying to run the tools/doc/node_modules, and .docbuildstamp make targets: internal/util.js:97 throw new ERR_NO_CRYPTO(); ^ Error [ERR_NO_CRYPTO]: Node.js is not compiled with OpenSSL crypto support at assertCrypto (internal/util.js:97:11) at crypto.js:31:1 ... at Object. (/node/deps/npm/node_modules/uuid/lib/rng.js:4:14) at Module._compile (internal/modules/cjs/loader.js:689:30) ... make[1]: *** [tools/doc/node_modules] Error 1 This commit adds crypto check to these targets to allow the build to pass. PR-URL: https://github.com/nodejs/node/pull/22148 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca Reviewed-By: Colin Ihrig Reviewed-By: Michael Dawson Reviewed-By: Richard Lau --- Makefile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f1242bbb30fc..0869a8067b50 100644 --- a/Makefile +++ b/Makefile @@ -320,10 +320,16 @@ ifeq ($(OSTYPE),aix) DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp endif +node_use_openssl = $(shell $(call available-node,"-p" \ + "process.versions.openssl != undefined")) test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules +ifeq ($(node_use_openssl),true) $(RM) -r test/addons/??_*/ [ -x $(NODE) ] && $(NODE) $< || node $< touch $@ +else + @echo "Skipping .docbuildstamp (no crypto)" +endif ADDONS_BINDING_GYPS := \ $(filter-out test/addons/??_*/binding.gyp, \ @@ -1062,15 +1068,17 @@ lint-md-build: tools/remark-cli/node_modules \ .PHONY: tools/doc/node_modules tools/doc/node_modules: - @cd tools/doc && $(call available-node,$(run-npm-install)) +ifeq ($(node_use_openssl),true) + cd tools/doc && $(call available-node,$(run-npm-install)) +else + @echo "Skipping tools/doc/node_modules (no crypto)" +endif .PHONY: lint-md ifneq ("","$(wildcard tools/remark-cli/node_modules/)") LINT_MD_DOC_FILES = $(shell ls doc/*.md doc/**/*.md) run-lint-doc-md = tools/remark-cli/cli.js -q -f $(LINT_MD_DOC_FILES) -node_use_openssl = $(shell $(call available-node,"-p" \ - "process.versions.openssl != undefined")) # Lint all changed markdown files under doc/ tools/.docmdlintstamp: $(LINT_MD_DOC_FILES) ifeq ($(node_use_openssl),true) From 94840fdb9db2a624bb3e071c462f2a8607a995bd Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Mon, 6 Aug 2018 08:54:04 +0200 Subject: [PATCH 03/17] test: move require of http2 to after crypto check Currently when configured without-ssl test-heapdump-http2.js will fail with a missing crypto message. This commit moves the require of http2 to after the crypto check. PR-URL: https://github.com/nodejs/node/pull/22148 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca Reviewed-By: Colin Ihrig Reviewed-By: Michael Dawson Reviewed-By: Richard Lau --- test/parallel/test-heapdump-http2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-heapdump-http2.js b/test/parallel/test-heapdump-http2.js index cbc1209ab055..b503951e6585 100644 --- a/test/parallel/test-heapdump-http2.js +++ b/test/parallel/test-heapdump-http2.js @@ -2,9 +2,9 @@ 'use strict'; const common = require('../common'); const { recordState } = require('../common/heap'); -const http2 = require('http2'); if (!common.hasCrypto) common.skip('missing crypto'); +const http2 = require('http2'); { const state = recordState(); From 346f2a7bec5ad60a3e4d7af3b0ca83e63207ede1 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Mon, 6 Aug 2018 08:54:04 +0200 Subject: [PATCH 04/17] test: move require of https to after crypto check Currently when configured without-ssl test-request-arguments.js will fail with a missing crypto message. This commit moves the require of https to after the crypto check. PR-URL: https://github.com/nodejs/node/pull/22148 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca Reviewed-By: Colin Ihrig Reviewed-By: Michael Dawson Reviewed-By: Richard Lau --- test/parallel/test-https-request-arguments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-https-request-arguments.js b/test/parallel/test-https-request-arguments.js index 44037ddd6de9..9dc80094be0d 100644 --- a/test/parallel/test-https-request-arguments.js +++ b/test/parallel/test-https-request-arguments.js @@ -1,11 +1,11 @@ 'use strict'; const common = require('../common'); const assert = require('assert'); -const https = require('https'); const fixtures = require('../common/fixtures'); if (!common.hasCrypto) common.skip('missing crypto'); +const https = require('https'); const options = { key: fixtures.readKey('agent1-key.pem'), From 34f56e2d711aaff10838167aa4ece77d278cfa09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Nie=C3=9Fen?= Date: Sat, 4 Aug 2018 18:39:45 +0200 Subject: [PATCH 05/17] test: fix crypto test case PR-URL: https://github.com/nodejs/node/pull/22126 Reviewed-By: Ujjwal Sharma Reviewed-By: Matteo Collina Reviewed-By: Trivikram Kamat Reviewed-By: James M Snell --- test/parallel/test-crypto-cipher-decipher.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/parallel/test-crypto-cipher-decipher.js b/test/parallel/test-crypto-cipher-decipher.js index 81c5f32d4928..3c0658762f47 100644 --- a/test/parallel/test-crypto-cipher-decipher.js +++ b/test/parallel/test-crypto-cipher-decipher.js @@ -112,15 +112,6 @@ testCipher2(Buffer.from('0123456789abcdef')); 'TypedArray, or DataView. Received type object' }); - common.expectsError( - () => crypto.createCipher('aes-256-cbc', 'secret').setAuthTag(null), - { - code: 'ERR_INVALID_ARG_TYPE', - type: TypeError, - message: 'The "buffer" argument must be one of type Buffer, ' + - 'TypedArray, or DataView. Received type object' - }); - common.expectsError( () => crypto.createCipher('aes-256-cbc', 'secret').setAAD(null), { @@ -146,6 +137,15 @@ testCipher2(Buffer.from('0123456789abcdef')); 'Received type object' }); + common.expectsError( + () => crypto.createDecipher('aes-256-cbc', 'secret').setAuthTag(null), + { + code: 'ERR_INVALID_ARG_TYPE', + type: TypeError, + message: 'The "buffer" argument must be one of type Buffer, ' + + 'TypedArray, or DataView. Received type object' + }); + common.expectsError( () => crypto.createDecipher('aes-256-cbc', null), { From 39dd3a443064a6bce0e919aee04903398f16c39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Nie=C3=9Fen?= Date: Sat, 4 Aug 2018 18:13:05 +0200 Subject: [PATCH 06/17] crypto: deprecate useless crypto APIs The APIs were probably exposed by accident. getAuthTag and setAuthTag are not a usual getter/setter pair: Getting the authentication tag only makes sense in the context of encryption, setting it only makes sense in the context of decryption. Currently, both functions throw. Neither has been documented publicly. PR-URL: https://github.com/nodejs/node/pull/22126 Reviewed-By: Ujjwal Sharma Reviewed-By: Matteo Collina Reviewed-By: Trivikram Kamat Reviewed-By: James M Snell --- doc/api/deprecations.md | 10 +++++++ lib/internal/crypto/cipher.js | 32 ++++++++++++++++++---- test/parallel/test-crypto-authenticated.js | 21 -------------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md index 6262fe4cf54e..56935574e618 100644 --- a/doc/api/deprecations.md +++ b/doc/api/deprecations.md @@ -1021,6 +1021,16 @@ accessed outside of Node.js core: `Socket.prototype._handle`, `Socket.prototype._healthCheck()`, `Socket.prototype._stopReceiving()`, and `dgram._createSocketHandle()`. + +### DEP0113: Cipher.setAuthTag(), Decipher.getAuthTag() + +Type: Runtime + +With the current crypto API, having `Cipher.setAuthTag()` and +`Decipher.getAuthTag()` is not helpful and both functions will throw an error +when called. They have never been documented and will be removed in a future +release. + [`--pending-deprecation`]: cli.html#cli_pending_deprecation [`Buffer.allocUnsafeSlow(size)`]: buffer.html#buffer_class_method_buffer_allocunsafeslow_size [`Buffer.from(array)`]: buffer.html#buffer_class_method_buffer_from_array diff --git a/lib/internal/crypto/cipher.js b/lib/internal/crypto/cipher.js index 94acc4063910..1828b69b140a 100644 --- a/lib/internal/crypto/cipher.js +++ b/lib/internal/crypto/cipher.js @@ -31,7 +31,7 @@ const assert = require('assert'); const LazyTransform = require('internal/streams/lazy_transform'); const { inherits } = require('util'); -const { normalizeEncoding } = require('internal/util'); +const { deprecate, normalizeEncoding } = require('internal/util'); // Lazy loaded for startup performance. let StringDecoder; @@ -194,7 +194,7 @@ Cipher.prototype.getAuthTag = function getAuthTag() { }; -Cipher.prototype.setAuthTag = function setAuthTag(tagbuf) { +function setAuthTag(tagbuf) { if (!isArrayBufferView(tagbuf)) { throw new ERR_INVALID_ARG_TYPE('buffer', ['Buffer', 'TypedArray', 'DataView'], @@ -203,7 +203,14 @@ Cipher.prototype.setAuthTag = function setAuthTag(tagbuf) { if (!this._handle.setAuthTag(tagbuf)) throw new ERR_CRYPTO_INVALID_STATE('setAuthTag'); return this; -}; +} + +Object.defineProperty(Cipher.prototype, 'setAuthTag', { + get: deprecate(() => setAuthTag, + 'Cipher.setAuthTag is deprecated and will be removed in a ' + + 'future version of Node.js.', + 'DEP0113') +}); Cipher.prototype.setAAD = function setAAD(aadbuf, options) { if (!isArrayBufferView(aadbuf)) { @@ -231,8 +238,23 @@ function addCipherPrototypeFunctions(constructor) { constructor.prototype.update = Cipher.prototype.update; constructor.prototype.final = Cipher.prototype.final; constructor.prototype.setAutoPadding = Cipher.prototype.setAutoPadding; - constructor.prototype.getAuthTag = Cipher.prototype.getAuthTag; - constructor.prototype.setAuthTag = Cipher.prototype.setAuthTag; + if (constructor === Cipheriv) { + constructor.prototype.getAuthTag = Cipher.prototype.getAuthTag; + Object.defineProperty(constructor.prototype, 'setAuthTag', { + get: deprecate(() => setAuthTag, + 'Cipher.setAuthTag is deprecated and will be removed in ' + + 'a future version of Node.js.', + 'DEP0113') + }); + } else { + constructor.prototype.setAuthTag = setAuthTag; + Object.defineProperty(constructor.prototype, 'getAuthTag', { + get: deprecate(() => constructor.prototype.getAuthTag, + 'Decipher.getAuthTag is deprecated and will be removed ' + + 'in a future version of Node.js.', + 'DEP0113') + }); + } constructor.prototype.setAAD = Cipher.prototype.setAAD; } diff --git a/test/parallel/test-crypto-authenticated.js b/test/parallel/test-crypto-authenticated.js index c7e89d6244d2..5c0fbb6a95fd 100644 --- a/test/parallel/test-crypto-authenticated.js +++ b/test/parallel/test-crypto-authenticated.js @@ -207,27 +207,6 @@ for (const test of TEST_CASES) { assert.throws(function() { encrypt.getAuthTag(); }, errMessages.state); } - { - // trying to set tag on encryption object: - const encrypt = crypto.createCipheriv(test.algo, - Buffer.from(test.key, 'hex'), - Buffer.from(test.iv, 'hex'), - options); - assert.throws(() => { encrypt.setAuthTag(Buffer.from(test.tag, 'hex')); }, - errMessages.state); - } - - { - if (!isCCM || !common.hasFipsCrypto) { - // trying to read tag from decryption object: - const decrypt = crypto.createDecipheriv(test.algo, - Buffer.from(test.key, 'hex'), - Buffer.from(test.iv, 'hex'), - options); - assert.throws(function() { decrypt.getAuthTag(); }, errMessages.state); - } - } - { // trying to create cipher with incorrect IV length assert.throws(function() { From 88bff82624628df261a429386d81565023062e44 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 8 Aug 2018 07:41:40 +0200 Subject: [PATCH 07/17] build: make tools/doc/node_modules non-phony This commit makes the target tools/doc/node_modules a non-phony target and also adds tools/doc/package.json as a prerequisite to it to avoid running it unnecessary. This is currently causing the target test/addons/.docbuildstamp to be always be executed as it has tools/doc/node_modules as a prerequisite. PR-URL: https://github.com/nodejs/node/pull/22189 Reviewed-By: Richard Lau Reviewed-By: Franziska Hinkelmann Reviewed-By: Sam Ruby Reviewed-By: Joyee Cheung Reviewed-By: Refael Ackermann --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0869a8067b50..525e28e5efbb 100644 --- a/Makefile +++ b/Makefile @@ -1066,8 +1066,7 @@ lint-md-build: tools/remark-cli/node_modules \ tools/doc/node_modules \ tools/remark-preset-lint-node/node_modules -.PHONY: tools/doc/node_modules -tools/doc/node_modules: +tools/doc/node_modules: tools/doc/package.json ifeq ($(node_use_openssl),true) cd tools/doc && $(call available-node,$(run-npm-install)) else From c85933cbd07ec3e32fdb27c4c4ef1aeadc609345 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 4 Aug 2018 09:48:50 -0700 Subject: [PATCH 08/17] trace_events,async_hooks: use intrinsic trace Switch to using the intrinsic trace event method for async_hooks. This is a breaking change because of the switch to a nested data argument for exec id and trigger id values. PR-URL: https://github.com/nodejs/node/pull/22127 Reviewed-By: Andreas Madsen Reviewed-By: Matteo Collina Reviewed-By: Ali Ijaz Sheikh --- lib/internal/trace_events_async_hooks.js | 20 +++++++++---------- src/async_wrap.cc | 20 ++++++++++++------- .../parallel/test-trace-events-async-hooks.js | 4 ++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/lib/internal/trace_events_async_hooks.js b/lib/internal/trace_events_async_hooks.js index 011dc6bbead1..f687f1ced8e1 100644 --- a/lib/internal/trace_events_async_hooks.js +++ b/lib/internal/trace_events_async_hooks.js @@ -1,6 +1,7 @@ 'use strict'; exports.setup = function(traceEvents, traceEventCategory) { + const { trace } = traceEvents; const async_wrap = process.binding('async_wrap'); const async_hooks = require('async_hooks'); @@ -27,34 +28,33 @@ exports.setup = function(traceEvents, traceEventCategory) { if (nativeProviders.has(type)) return; typeMemory.set(asyncId, type); - traceEvents.emit(BEFORE_EVENT, traceEventCategory, - type, asyncId, - 'triggerAsyncId', triggerAsyncId, - 'executionAsyncId', async_hooks.executionAsyncId()); + trace(BEFORE_EVENT, traceEventCategory, + type, asyncId, + { + triggerAsyncId, + executionAsyncId: async_hooks.executionAsyncId() + }); }, before(asyncId) { const type = typeMemory.get(asyncId); if (type === undefined) return; - traceEvents.emit(BEFORE_EVENT, traceEventCategory, - type + '_CALLBACK', asyncId); + trace(BEFORE_EVENT, traceEventCategory, `${type}_CALLBACK`, asyncId); }, after(asyncId) { const type = typeMemory.get(asyncId); if (type === undefined) return; - traceEvents.emit(END_EVENT, traceEventCategory, - type + '_CALLBACK', asyncId); + trace(END_EVENT, traceEventCategory, `${type}_CALLBACK`, asyncId); }, destroy(asyncId) { const type = typeMemory.get(asyncId); if (type === undefined) return; - traceEvents.emit(END_EVENT, traceEventCategory, - type, asyncId); + trace(END_EVENT, traceEventCategory, type, asyncId); // cleanup asyncId to type map typeMemory.delete(asyncId); diff --git a/src/async_wrap.cc b/src/async_wrap.cc index 7f2a0ece2f00..51072bb8c9c8 100644 --- a/src/async_wrap.cc +++ b/src/async_wrap.cc @@ -23,6 +23,7 @@ #include "env-inl.h" #include "node_internals.h" #include "util-inl.h" +#include "tracing/traced_value.h" #include "v8.h" #include "v8-profiler.h" @@ -608,13 +609,18 @@ void AsyncWrap::AsyncReset(double execution_async_id, bool silent) { switch (provider_type()) { #define V(PROVIDER) \ case PROVIDER_ ## PROVIDER: \ - TRACE_EVENT_NESTABLE_ASYNC_BEGIN2( \ - TRACING_CATEGORY_NODE1(async_hooks), \ - #PROVIDER, static_cast(get_async_id()), \ - "executionAsyncId", \ - static_cast(env()->execution_async_id()), \ - "triggerAsyncId", \ - static_cast(get_trigger_async_id())); \ + if (*TRACE_EVENT_API_GET_CATEGORY_GROUP_ENABLED( \ + TRACING_CATEGORY_NODE1(async_hooks))) { \ + auto data = tracing::TracedValue::Create(); \ + data->SetInteger("executionAsyncId", \ + static_cast(env()->execution_async_id())); \ + data->SetInteger("triggerAsyncId", \ + static_cast(get_trigger_async_id())); \ + TRACE_EVENT_NESTABLE_ASYNC_BEGIN1( \ + TRACING_CATEGORY_NODE1(async_hooks), \ + #PROVIDER, static_cast(get_async_id()), \ + "data", std::move(data)); \ + } \ break; NODE_ASYNC_PROVIDER_TYPES(V) #undef V diff --git a/test/parallel/test-trace-events-async-hooks.js b/test/parallel/test-trace-events-async-hooks.js index 13cd523de7ae..7c82eb9364a2 100644 --- a/test/parallel/test-trace-events-async-hooks.js +++ b/test/parallel/test-trace-events-async-hooks.js @@ -61,8 +61,8 @@ proc.once('exit', common.mustCall(() => { return (trace.ph === 'b' && !trace.name.includes('_CALLBACK')); }); assert.ok(initEvents.every((trace) => { - return (trace.args.executionAsyncId > 0 && - trace.args.triggerAsyncId > 0); + return (trace.args.data.executionAsyncId > 0 && + trace.args.data.triggerAsyncId > 0); }), `Unexpected initEvents format: ${util.inspect(initEvents)}`); })); })); From b85460498fc24b855efbc2516f8e7cc629e24bb6 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 9 Aug 2018 13:44:44 -0700 Subject: [PATCH 09/17] src: remove old process.binding('trace_events').emit Remove the older emit and categoryGroupEnabled bindings in favor of the new intrinsics PR-URL: https://github.com/nodejs/node/pull/22127 Reviewed-By: Andreas Madsen Reviewed-By: Matteo Collina Reviewed-By: Ali Ijaz Sheikh --- benchmark/misc/trace.js | 29 +--- lib/internal/bootstrap/node.js | 2 +- src/node_trace_events.cc | 126 ------------------ test/parallel/test-trace-events-binding.js | 22 +-- .../test-trace-events-category-used.js | 10 +- 5 files changed, 22 insertions(+), 167 deletions(-) diff --git a/benchmark/misc/trace.js b/benchmark/misc/trace.js index 7ac80ca4cf85..fb51c91e9e3e 100644 --- a/benchmark/misc/trace.js +++ b/benchmark/misc/trace.js @@ -4,7 +4,7 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { n: [100000], - method: ['trace', 'emit', 'isTraceCategoryEnabled', 'categoryGroupEnabled'] + method: ['trace', 'isTraceCategoryEnabled'] }, { flags: ['--expose-internals', '--trace-event-categories', 'foo'] }); @@ -13,14 +13,6 @@ const { TRACE_EVENT_PHASE_NESTABLE_ASYNC_BEGIN: kBeforeEvent } = process.binding('constants').trace; -function doEmit(n, emit) { - bench.start(); - for (var i = 0; i < n; i++) { - emit(kBeforeEvent, 'foo', 'test', 0, 'arg1', 1); - } - bench.end(n); -} - function doTrace(n, trace) { bench.start(); for (var i = 0; i < n; i++) { @@ -38,23 +30,12 @@ function doIsTraceCategoryEnabled(n, isTraceCategoryEnabled) { bench.end(n); } -function doCategoryGroupEnabled(n, categoryGroupEnabled) { - bench.start(); - for (var i = 0; i < n; i++) { - categoryGroupEnabled('foo'); - categoryGroupEnabled('bar'); - } - bench.end(n); -} - function main({ n, method }) { const { internalBinding } = require('internal/test/binding'); const { trace, - isTraceCategoryEnabled, - emit, - categoryGroupEnabled + isTraceCategoryEnabled } = internalBinding('trace_events'); switch (method) { @@ -62,15 +43,9 @@ function main({ n, method }) { case 'trace': doTrace(n, trace); break; - case 'emit': - doEmit(n, emit); - break; case 'isTraceCategoryEnabled': doIsTraceCategoryEnabled(n, isTraceCategoryEnabled); break; - case 'categoryGroupEnabled': - doCategoryGroupEnabled(n, categoryGroupEnabled); - break; default: throw new Error(`Unexpected method "${method}"`); } diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js index 5cd00932bd86..7299e71db529 100644 --- a/lib/internal/bootstrap/node.js +++ b/lib/internal/bootstrap/node.js @@ -103,7 +103,7 @@ const traceEvents = internalBinding('trace_events'); const traceEventCategory = 'node,node.async_hooks'; - if (traceEvents.categoryGroupEnabled(traceEventCategory)) { + if (traceEvents.isTraceCategoryEnabled(traceEventCategory)) { NativeModule.require('internal/trace_events_async_hooks') .setup(traceEvents, traceEventCategory); } diff --git a/src/node_trace_events.cc b/src/node_trace_events.cc index 8413620e3f5f..219e142a3195 100644 --- a/src/node_trace_events.cc +++ b/src/node_trace_events.cc @@ -13,7 +13,6 @@ using v8::Array; using v8::Context; using v8::FunctionCallbackInfo; using v8::FunctionTemplate; -using v8::Int32; using v8::Local; using v8::Object; using v8::String; @@ -99,137 +98,12 @@ void GetEnabledCategories(const FunctionCallbackInfo& args) { } } -// The tracing APIs require category groups to be pointers to long-lived -// strings. Those strings are stored here. -static std::unordered_set category_groups; -static Mutex category_groups_mutex; - -// Gets a pointer to the category-enabled flags for a tracing category group, -// if tracing is enabled for it. -static const uint8_t* GetCategoryGroupEnabled(const char* category_group) { - CHECK_NOT_NULL(category_group); - return TRACE_EVENT_API_GET_CATEGORY_GROUP_ENABLED(category_group); -} - -static const char* GetCategoryGroup(Environment* env, - const Local category_value) { - CHECK(category_value->IsString()); - - Utf8Value category(env->isolate(), category_value); - Mutex::ScopedLock lock(category_groups_mutex); - // If the value already exists in the set, insertion.first will point - // to the existing value. Thus, this will maintain a long lived pointer - // to the category c-string. - auto insertion = category_groups.insert(category.out()); - - // The returned insertion is a pair whose first item is the object that was - // inserted or that blocked the insertion and second item is a boolean - // indicating whether it was inserted. - return insertion.first->c_str(); -} - -static void Emit(const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - Local context = env->context(); - - // Args: [type, category, name, id, argName, argValue] - CHECK_GE(args.Length(), 3); - - // Check the category group first, to avoid doing more work if it's not - // enabled. - const char* category_group = GetCategoryGroup(env, args[1]); - const uint8_t* category_group_enabled = - GetCategoryGroupEnabled(category_group); - if (*category_group_enabled == 0) return; - - // get trace_event phase - CHECK(args[0]->IsNumber()); - char phase = static_cast(args[0]->Int32Value(context).ToChecked()); - - // get trace_event name - CHECK(args[2]->IsString()); - Utf8Value name_value(env->isolate(), args[2]); - const char* name = name_value.out(); - - // get trace_event id - int64_t id = 0; - bool has_id = false; - if (args.Length() >= 4 && !args[3]->IsUndefined() && !args[3]->IsNull()) { - has_id = true; - CHECK(args[3]->IsNumber()); - id = args[3]->IntegerValue(context).ToChecked(); - } - - // TODO(AndreasMadsen): String values are not supported. - int32_t num_args = 0; - const char* arg_names[2]; - uint8_t arg_types[2]; - uint64_t arg_values[2]; - - // Create Utf8Value in the function scope to prevent deallocation. - // The c-string will be copied by TRACE_EVENT_API_ADD_TRACE_EVENT at the end. - Utf8Value arg1NameValue(env->isolate(), args[4]); - Utf8Value arg2NameValue(env->isolate(), args[6]); - - if (args.Length() >= 6 && - (!args[4]->IsUndefined() || !args[5]->IsUndefined())) { - num_args = 1; - arg_types[0] = TRACE_VALUE_TYPE_INT; - - CHECK(args[4]->IsString()); - arg_names[0] = arg1NameValue.out(); - - CHECK(args[5]->IsNumber()); - arg_values[0] = args[5]->IntegerValue(context).ToChecked(); - } - - if (args.Length() >= 8 && - (!args[6]->IsUndefined() || !args[7]->IsUndefined())) { - num_args = 2; - arg_types[1] = TRACE_VALUE_TYPE_INT; - - CHECK(args[6]->IsString()); - arg_names[1] = arg2NameValue.out(); - - CHECK(args[7]->IsNumber()); - arg_values[1] = args[7]->IntegerValue(context).ToChecked(); - } - - // The TRACE_EVENT_FLAG_COPY flag indicates that the name and argument - // name should be copied thus they don't need to long-lived pointers. - // The category name still won't be copied and thus need to be a long-lived - // pointer. - uint32_t flags = TRACE_EVENT_FLAG_COPY; - if (has_id) { - flags |= TRACE_EVENT_FLAG_HAS_ID; - } - - const char* scope = node::tracing::kGlobalScope; - uint64_t bind_id = node::tracing::kNoId; - - TRACE_EVENT_API_ADD_TRACE_EVENT( - phase, category_group_enabled, name, scope, id, bind_id, - num_args, arg_names, arg_types, arg_values, - flags); -} - -static void CategoryGroupEnabled(const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - - const char* category_group = GetCategoryGroup(env, args[0]); - const uint8_t* category_group_enabled = - GetCategoryGroupEnabled(category_group); - args.GetReturnValue().Set(*category_group_enabled > 0); -} - void Initialize(Local target, Local unused, Local context, void* priv) { Environment* env = Environment::GetCurrent(context); - env->SetMethod(target, "emit", Emit); - env->SetMethod(target, "categoryGroupEnabled", CategoryGroupEnabled); env->SetMethod(target, "getEnabledCategories", GetEnabledCategories); Local category_set = diff --git a/test/parallel/test-trace-events-binding.js b/test/parallel/test-trace-events-binding.js index 2544c196acee..163d9c4cbd62 100644 --- a/test/parallel/test-trace-events-binding.js +++ b/test/parallel/test-trace-events-binding.js @@ -9,15 +9,14 @@ if (!common.isMainThread) const CODE = ` const { internalBinding } = require('internal/test/binding'); - const { emit } = internalBinding('trace_events'); - emit('b'.charCodeAt(0), 'custom', - 'type-value', 10, 'extra-value', 20); - emit('b'.charCodeAt(0), 'custom', - 'type-value', 20, 'first-value', 20, 'second-value', 30); - emit('b'.charCodeAt(0), 'custom', - 'type-value', 30); - emit('b'.charCodeAt(0), 'missing', - 'type-value', 10, 'extra-value', 20); + const { trace } = internalBinding('trace_events'); + trace('b'.charCodeAt(0), 'custom', + 'type-value', 10, {'extra-value': 20 }); + trace('b'.charCodeAt(0), 'custom', + 'type-value', 20, {'first-value': 20, 'second-value': 30 }); + trace('b'.charCodeAt(0), 'custom', 'type-value', 30); + trace('b'.charCodeAt(0), 'missing', + 'type-value', 10, {'extra-value': 20 }); `; const FILE_NAME = 'node_trace.1.log'; @@ -27,6 +26,7 @@ process.chdir(tmpdir.path); const proc = cp.spawn(process.execPath, [ '--trace-event-categories', 'custom', + '--no-warnings', '--expose-internals', '-e', CODE ]); @@ -42,14 +42,14 @@ proc.once('exit', common.mustCall(() => { assert.strictEqual(traces[0].cat, 'custom'); assert.strictEqual(traces[0].name, 'type-value'); assert.strictEqual(traces[0].id, '0xa'); - assert.deepStrictEqual(traces[0].args, { 'extra-value': 20 }); + assert.deepStrictEqual(traces[0].args.data, { 'extra-value': 20 }); assert.strictEqual(traces[1].pid, proc.pid); assert.strictEqual(traces[1].ph, 'b'); assert.strictEqual(traces[1].cat, 'custom'); assert.strictEqual(traces[1].name, 'type-value'); assert.strictEqual(traces[1].id, '0x14'); - assert.deepStrictEqual(traces[1].args, { + assert.deepStrictEqual(traces[1].args.data, { 'first-value': 20, 'second-value': 30 }); diff --git a/test/parallel/test-trace-events-category-used.js b/test/parallel/test-trace-events-category-used.js index 4f9ad69366fd..249ccd4aa3a3 100644 --- a/test/parallel/test-trace-events-category-used.js +++ b/test/parallel/test-trace-events-category-used.js @@ -8,9 +8,9 @@ if (!common.isMainThread) const CODE = ` const { internalBinding } = require('internal/test/binding'); - const { categoryGroupEnabled } = internalBinding('trace_events'); + const { isTraceCategoryEnabled } = internalBinding('trace_events'); console.log( - categoryGroupEnabled("custom") + isTraceCategoryEnabled("custom") ); `; @@ -21,6 +21,9 @@ process.chdir(tmpdir.path); const procEnabled = cp.spawn( process.execPath, [ '--trace-event-categories', 'custom', + // make test less noisy since internal/test/binding + // emits a warning. + '--no-warnings', '--expose-internals', '-e', CODE ] ); @@ -35,6 +38,9 @@ procEnabled.once('exit', common.mustCall(() => { const procDisabled = cp.spawn( process.execPath, [ '--trace-event-categories', 'other', + // make test less noisy since internal/test/binding + // emits a warning. + '--no-warnings', '--expose-internals', '-e', CODE ] ); From fe47b8b6a529233ada74dfb979e6164111737fc2 Mon Sep 17 00:00:00 2001 From: Denys Otrishko Date: Fri, 6 Jul 2018 02:19:15 +0300 Subject: [PATCH 10/17] stream: fix readable behavior for highWaterMark === 0 Avoid trying to emit 'readable' due to the fact that state.length is always >= state.highWaterMark if highWaterMark is 0. Therefore upon .read(0) call (through .on('readable')) stream assumed that it has enough data to emit 'readable' even though state.length === 0 instead of issuing _read(). Which led to the TTY not recognizing that someone is waiting for the input. Fixes: https://github.com/nodejs/node/issues/20503 Refs: https://github.com/nodejs/node/pull/18372 PR-URL: https://github.com/nodejs/node/pull/21690 Reviewed-By: Matteo Collina Reviewed-By: Ruben Bridgewater --- lib/_stream_readable.js | 6 +++- test/parallel/test-readable-single-end.js | 16 ++++++++++ test/parallel/test-stream-readable-hwm-0.js | 30 +++++++++++++++++++ .../pseudo-tty/test-readable-tty-keepalive.js | 29 ++++++++++++++++++ .../test-readable-tty-keepalive.out | 0 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 test/parallel/test-readable-single-end.js create mode 100644 test/parallel/test-stream-readable-hwm-0.js create mode 100644 test/pseudo-tty/test-readable-tty-keepalive.js create mode 100644 test/pseudo-tty/test-readable-tty-keepalive.out diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 0b5cac6b8376..029faaa6142d 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -383,7 +383,10 @@ Readable.prototype.read = function(n) { // the 'readable' event and move on. if (n === 0 && state.needReadable && - (state.length >= state.highWaterMark || state.ended)) { + ((state.highWaterMark !== 0 ? + state.length >= state.highWaterMark : + state.length > 0) || + state.ended)) { debug('read: emitReadable', state.length, state.ended); if (state.length === 0 && state.ended) endReadable(this); @@ -808,6 +811,7 @@ Readable.prototype.on = function(ev, fn) { if (!state.endEmitted && !state.readableListening) { state.readableListening = state.needReadable = true; state.emittedReadable = false; + debug('on readable', state.length, state.reading); if (state.length) { emitReadable(this); } else if (!state.reading) { diff --git a/test/parallel/test-readable-single-end.js b/test/parallel/test-readable-single-end.js new file mode 100644 index 000000000000..0969d49aa48e --- /dev/null +++ b/test/parallel/test-readable-single-end.js @@ -0,0 +1,16 @@ +'use strict'; + +const common = require('../common'); +const { Readable } = require('stream'); + +// This test ensures that there will not be an additional empty 'readable' +// event when stream has ended (only 1 event signalling about end) + +const r = new Readable({ + read: () => {}, +}); + +r.push(null); + +r.on('readable', common.mustCall()); +r.on('end', common.mustCall()); diff --git a/test/parallel/test-stream-readable-hwm-0.js b/test/parallel/test-stream-readable-hwm-0.js new file mode 100644 index 000000000000..ecbc197d4820 --- /dev/null +++ b/test/parallel/test-stream-readable-hwm-0.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common'); + +// This test ensures that Readable stream will call _read() for streams +// with highWaterMark === 0 upon .read(0) instead of just trying to +// emit 'readable' event. + +const assert = require('assert'); +const { Readable } = require('stream'); + +const r = new Readable({ + // must be called only once upon setting 'readable' listener + read: common.mustCall(), + highWaterMark: 0, +}); + +let pushedNull = false; +// this will trigger read(0) but must only be called after push(null) +// because the we haven't pushed any data +r.on('readable', common.mustCall(() => { + assert.strictEqual(r.read(), null); + assert.strictEqual(pushedNull, true); +})); +r.on('end', common.mustCall()); +process.nextTick(() => { + assert.strictEqual(r.read(), null); + pushedNull = true; + r.push(null); +}); diff --git a/test/pseudo-tty/test-readable-tty-keepalive.js b/test/pseudo-tty/test-readable-tty-keepalive.js new file mode 100644 index 000000000000..4cf0b9fb2963 --- /dev/null +++ b/test/pseudo-tty/test-readable-tty-keepalive.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); + +// This test ensures that Node.js will not ignore tty 'readable' subscribers +// when it's the only tty subscriber and the only thing keeping event loop alive +// https://github.com/nodejs/node/issues/20503 + +process.stdin.setEncoding('utf8'); + +const expectedInput = ['foo', 'bar', null]; + +process.stdin.on('readable', common.mustCall(function() { + const data = process.stdin.read(); + assert.strictEqual(data, expectedInput.shift()); +}, 3)); // first 2 data, then end + +process.stdin.on('end', common.mustCall()); + +setTimeout(() => { + process.stdin.push('foo'); + process.nextTick(() => { + process.stdin.push('bar'); + process.nextTick(() => { + process.stdin.push(null); + }); + }); +}, 1); diff --git a/test/pseudo-tty/test-readable-tty-keepalive.out b/test/pseudo-tty/test-readable-tty-keepalive.out new file mode 100644 index 000000000000..e69de29bb2d1 From 91e897ac9281de1a36139b6514609753f1154ab2 Mon Sep 17 00:00:00 2001 From: George Adams Date: Fri, 10 Aug 2018 16:41:49 +0100 Subject: [PATCH 11/17] doc: add gdams to collaborators PR-URL: https://github.com/nodejs/node/pull/22236 Reviewed-By: Anna Henningsen Reviewed-By: Richard Lau Reviewed-By: Refael Ackermann Reviewed-By: Michael Dawson Reviewed-By: James M Snell Reviewed-By: Vse Mozhet Byt --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 925c9e39b8c1..f7ef7c66b191 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,8 @@ For more information about the governance of the Node.js project, see **Jeremiah Senkpiel** <fishrock123@rocketmail.com> * [gabrielschulhof](https://github.com/gabrielschulhof) - **Gabriel Schulhof** <gabriel.schulhof@intel.com> +* [gdams](https://github.com/gdams) - +**George Adams** <george.adams@uk.ibm.com> (he/him) * [geek](https://github.com/geek) - **Wyatt Preul** <wpreul@gmail.com> * [gibfahn](https://github.com/gibfahn) - From dcfd323c6be53c6ea812e5d98eee8f4e8d31dc57 Mon Sep 17 00:00:00 2001 From: Musa Hamwala Date: Wed, 8 Aug 2018 10:37:21 +0100 Subject: [PATCH 12/17] test: remove second arg from assert.ifError() `test/parallel/test-fs-readfile.js` has a call to `assert.ifError()` that receives two arguments. There is no second argument used in `assert.ifError()`. This PR removes this argument. PR-URL: https://github.com/nodejs/node/pull/22190 Reviewed-By: George Adams Reviewed-By: Richard Lau Reviewed-By: Colin Ihrig Reviewed-By: Jon Moss Reviewed-By: Ruben Bridgewater Reviewed-By: Trivikram Kamat Reviewed-By: Luigi Pinca Reviewed-By: Rich Trott Reviewed-By: James M Snell --- test/parallel/test-fs-readfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-fs-readfile.js b/test/parallel/test-fs-readfile.js index 287834ab201e..118f2e43aab5 100644 --- a/test/parallel/test-fs-readfile.js +++ b/test/parallel/test-fs-readfile.js @@ -53,7 +53,7 @@ for (const e of fileInfo) { for (const e of fileInfo) { fs.readFile(e.name, common.mustCall((err, buf) => { console.log(`Validating readFile on file ${e.name} of length ${e.len}`); - assert.ifError(err, 'An error occurred'); + assert.ifError(err); assert.deepStrictEqual(buf, e.contents, 'Incorrect file contents'); })); } From 7b1f3a4133cf7481b36a1ae24cd4650730b4bab4 Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Thu, 9 Aug 2018 14:46:07 -0700 Subject: [PATCH 13/17] doc: add subprocess.ref() and subprocess.unref() PR-URL: https://github.com/nodejs/node/pull/22220 Reviewed-By: Anna Henningsen Reviewed-By: Bryan English Reviewed-By: James M Snell Reviewed-By: Tiancheng "Timothy" Gu Reviewed-By: Colin Ihrig Reviewed-By: Vse Mozhet Byt --- doc/api/child_process.md | 55 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/doc/api/child_process.md b/doc/api/child_process.md index a2e6bb5365cf..ee75cbc5f348 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -536,11 +536,12 @@ process will be made the leader of a new process group and session. Note that child processes may continue running after the parent exits regardless of whether they are detached or not. See setsid(2) for more information. -By default, the parent will wait for the detached child to exit. To prevent -the parent from waiting for a given `subprocess`, use the `subprocess.unref()` -method. Doing so will cause the parent's event loop to not include the child in -its reference count, allowing the parent to exit independently of the child, -unless there is an established IPC channel between the child and parent. +By default, the parent will wait for the detached child to exit. To prevent the +parent from waiting for a given `subprocess` to exit, use the +`subprocess.unref()` method. Doing so will cause the parent's event loop to not +include the child in its reference count, allowing the parent to exit +independently of the child, unless there is an established IPC channel between +the child and the parent. When using the `detached` option to start a long-running process, the process will not stay running in the background after the parent exits unless it is @@ -1094,6 +1095,27 @@ console.log(`Spawned child pid: ${grep.pid}`); grep.stdin.end(); ``` +### subprocess.ref() + + +Calling `subprocess.ref()` after making a call to `subprocess.unref()` will +restore the removed reference count for the child process, forcing the parent +to wait for the child to exit before exiting itself. + +```js +const { spawn } = require('child_process'); + +const subprocess = spawn(process.argv[0], ['child_program.js'], { + detached: true, + stdio: 'ignore' +}); + +subprocess.unref(); +subprocess.ref(); +``` + ### subprocess.send(message[, sendHandle[, options]][, callback]) + +By default, the parent will wait for the detached child to exit. To prevent the +parent from waiting for a given `subprocess` to exit, use the +`subprocess.unref()` method. Doing so will cause the parent's event loop to not +include the child in its reference count, allowing the parent to exit +independently of the child, unless there is an established IPC channel between +the child and the parent. + +```js +const { spawn } = require('child_process'); + +const subprocess = spawn(process.argv[0], ['child_program.js'], { + detached: true, + stdio: 'ignore' +}); + +subprocess.unref(); +``` + ## `maxBuffer` and Unicode The `maxBuffer` option specifies the largest number of bytes allowed on `stdout` From 3ce6bc3b5082478dfe6832997640c93de97705be Mon Sep 17 00:00:00 2001 From: Ouyang Yadong Date: Fri, 10 Aug 2018 10:51:24 +0800 Subject: [PATCH 14/17] dgram: remove unnecessary fd property from Socket It seems the property socket.fd is used long before for the compatibility reason but it's not used anymore. It's also not documented in docs so that it's not a public api/property. Keep the property also make the code a bit confusing as socket._handle.fd is the actual property representing fd. PR-URL: https://github.com/nodejs/node/pull/21684 Reviewed-By: Anna Henningsen Reviewed-By: Minwoo Jung Reviewed-By: Matteo Collina Reviewed-By: Colin Ihrig Reviewed-By: Jon Moss --- lib/dgram.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/dgram.js b/lib/dgram.js index 44a7f09d0061..af1e23f512d5 100644 --- a/lib/dgram.js +++ b/lib/dgram.js @@ -85,7 +85,6 @@ function Socket(type, listener) { this[async_id_symbol] = handle.getAsyncId(); this.type = type; - this.fd = null; // compatibility hack if (typeof listener === 'function') this.on('message', listener); @@ -116,7 +115,6 @@ function startListening(socket) { state.handle.recvStart(); state.receiving = true; state.bindState = BIND_STATE_BOUND; - socket.fd = -42; // compatibility hack if (state.recvBufferSize) bufferSize(socket, state.recvBufferSize, RECV_BUFFER); @@ -677,7 +675,6 @@ function stopReceiving(socket) { state.handle.recvStop(); state.receiving = false; - socket.fd = null; // compatibility hack } From 2e37d0fdc0ad4a23034b7a2793c766bbcc416de2 Mon Sep 17 00:00:00 2001 From: Ouyang Yadong Date: Wed, 8 Aug 2018 10:51:16 +0800 Subject: [PATCH 15/17] test: handle errors correctly in GC http test In test-gc-http-client-timeout.js, res.resume is not a function if error occurs. Remove the error handler. PR-URL: https://github.com/nodejs/node/pull/22185 Reviewed-By: Luigi Pinca Reviewed-By: Rich Trott --- test/parallel/test-gc-http-client-timeout.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-gc-http-client-timeout.js b/test/parallel/test-gc-http-client-timeout.js index e6878021ef9b..e5cb91c06f89 100644 --- a/test/parallel/test-gc-http-client-timeout.js +++ b/test/parallel/test-gc-http-client-timeout.js @@ -39,7 +39,7 @@ function getall() { pathname: '/', port: server.address().port }, cb); - req.on('error', cb); + req.setTimeout(10, function() { console.log('timeout (expected)'); }); From e0395247c899af101f8a1f76a8554be1ff14040a Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Wed, 25 Jul 2018 13:48:47 -0700 Subject: [PATCH 16/17] inspector: add inspector_protocol as a direct dependency Currently, node.js depends on inspector_protocol indirectly through the dependency on v8. This is a dependency violation that will make it hard to roll V8 into Node if V8 gets a newer inspector protocol version with incompatible API. In fact, this surfaced on one of our bots when we tried to roll new inspector_protocol into V8. This patch adds inspector protocol and its required dependencies to node deps: - jinja2 - markupsafe PR-URL: https://github.com/nodejs/node/pull/21975 Reviewed-By: Eugene Ostroukhov Reviewed-By: Aleksei Koziatinskii --- LICENSE | 103 + node.gyp | 8 +- .../CheckProtocolCompatibility.py | 479 +++++ tools/inspector_protocol/CodeGenerator.py | 654 +++++++ .../ConcatenateProtocols.py | 39 + .../ConvertProtocolToJSON.py | 183 ++ tools/inspector_protocol/LICENSE | 27 + tools/inspector_protocol/OWNERS | 8 + tools/inspector_protocol/README.v8 | 16 + .../inspector_protocol/inspector_protocol.gni | 89 + .../inspector_protocol.gypi | 33 + .../lib/Allocator_h.template | 30 + tools/inspector_protocol/lib/Array_h.template | 136 ++ .../lib/Collections_h.template | 43 + .../lib/DispatcherBase_cpp.template | 354 ++++ .../lib/DispatcherBase_h.template | 173 ++ .../lib/ErrorSupport_cpp.template | 71 + .../lib/ErrorSupport_h.template | 37 + .../inspector_protocol/lib/Forward_h.template | 37 + .../lib/FrontendChannel_h.template | 30 + tools/inspector_protocol/lib/Maybe_h.template | 86 + .../lib/Object_cpp.template | 38 + .../inspector_protocol/lib/Object_h.template | 32 + .../lib/Parser_cpp.template | 547 ++++++ .../inspector_protocol/lib/Parser_h.template | 22 + .../lib/Protocol_cpp.template | 12 + .../lib/ValueConversions_h.template | 171 ++ .../lib/Values_cpp.template | 409 ++++ .../inspector_protocol/lib/Values_h.template | 249 +++ .../templates/Exported_h.template | 65 + .../templates/Imported_h.template | 55 + .../templates/TypeBuilder_cpp.template | 397 ++++ .../templates/TypeBuilder_h.template | 301 +++ tools/jinja2/AUTHORS | 34 + tools/jinja2/Jinja2-2.10.tar.gz.md5 | 1 + tools/jinja2/Jinja2-2.10.tar.gz.sha512 | 1 + tools/jinja2/LICENSE | 31 + tools/jinja2/OWNERS | 6 + tools/jinja2/README.chromium | 26 + tools/jinja2/__init__.py | 83 + tools/jinja2/_compat.py | 99 + tools/jinja2/_identifier.py | 2 + tools/jinja2/asyncfilters.py | 146 ++ tools/jinja2/asyncsupport.py | 256 +++ tools/jinja2/bccache.py | 362 ++++ tools/jinja2/compiler.py | 1721 +++++++++++++++++ tools/jinja2/constants.py | 32 + tools/jinja2/debug.py | 372 ++++ tools/jinja2/defaults.py | 56 + tools/jinja2/environment.py | 1276 ++++++++++++ tools/jinja2/exceptions.py | 146 ++ tools/jinja2/ext.py | 627 ++++++ tools/jinja2/filters.py | 1190 ++++++++++++ tools/jinja2/get_jinja2.sh | 136 ++ tools/jinja2/idtracking.py | 286 +++ tools/jinja2/jinja2.gni | 31 + tools/jinja2/lexer.py | 739 +++++++ tools/jinja2/loaders.py | 481 +++++ tools/jinja2/meta.py | 106 + tools/jinja2/nativetypes.py | 220 +++ tools/jinja2/nodes.py | 999 ++++++++++ tools/jinja2/optimizer.py | 49 + tools/jinja2/parser.py | 903 +++++++++ tools/jinja2/runtime.py | 813 ++++++++ tools/jinja2/sandbox.py | 475 +++++ tools/jinja2/tests.py | 175 ++ tools/jinja2/utils.py | 647 +++++++ tools/jinja2/visitor.py | 87 + tools/license-builder.sh | 3 + tools/markupsafe/AUTHORS | 13 + tools/markupsafe/LICENSE | 33 + tools/markupsafe/MarkupSafe-0.18.tar.gz.md5 | 1 + .../markupsafe/MarkupSafe-0.18.tar.gz.sha512 | 1 + tools/markupsafe/OWNERS | 5 + tools/markupsafe/README.chromium | 24 + tools/markupsafe/__init__.py | 234 +++ tools/markupsafe/_compat.py | 24 + tools/markupsafe/_constants.py | 267 +++ tools/markupsafe/_native.py | 46 + tools/markupsafe/_speedups.c | 239 +++ tools/markupsafe/get_markupsafe.sh | 121 ++ 81 files changed, 18555 insertions(+), 4 deletions(-) create mode 100755 tools/inspector_protocol/CheckProtocolCompatibility.py create mode 100644 tools/inspector_protocol/CodeGenerator.py create mode 100755 tools/inspector_protocol/ConcatenateProtocols.py create mode 100644 tools/inspector_protocol/ConvertProtocolToJSON.py create mode 100644 tools/inspector_protocol/LICENSE create mode 100644 tools/inspector_protocol/OWNERS create mode 100644 tools/inspector_protocol/README.v8 create mode 100644 tools/inspector_protocol/inspector_protocol.gni create mode 100644 tools/inspector_protocol/inspector_protocol.gypi create mode 100644 tools/inspector_protocol/lib/Allocator_h.template create mode 100644 tools/inspector_protocol/lib/Array_h.template create mode 100644 tools/inspector_protocol/lib/Collections_h.template create mode 100644 tools/inspector_protocol/lib/DispatcherBase_cpp.template create mode 100644 tools/inspector_protocol/lib/DispatcherBase_h.template create mode 100644 tools/inspector_protocol/lib/ErrorSupport_cpp.template create mode 100644 tools/inspector_protocol/lib/ErrorSupport_h.template create mode 100644 tools/inspector_protocol/lib/Forward_h.template create mode 100644 tools/inspector_protocol/lib/FrontendChannel_h.template create mode 100644 tools/inspector_protocol/lib/Maybe_h.template create mode 100644 tools/inspector_protocol/lib/Object_cpp.template create mode 100644 tools/inspector_protocol/lib/Object_h.template create mode 100644 tools/inspector_protocol/lib/Parser_cpp.template create mode 100644 tools/inspector_protocol/lib/Parser_h.template create mode 100644 tools/inspector_protocol/lib/Protocol_cpp.template create mode 100644 tools/inspector_protocol/lib/ValueConversions_h.template create mode 100644 tools/inspector_protocol/lib/Values_cpp.template create mode 100644 tools/inspector_protocol/lib/Values_h.template create mode 100644 tools/inspector_protocol/templates/Exported_h.template create mode 100644 tools/inspector_protocol/templates/Imported_h.template create mode 100644 tools/inspector_protocol/templates/TypeBuilder_cpp.template create mode 100644 tools/inspector_protocol/templates/TypeBuilder_h.template create mode 100644 tools/jinja2/AUTHORS create mode 100644 tools/jinja2/Jinja2-2.10.tar.gz.md5 create mode 100644 tools/jinja2/Jinja2-2.10.tar.gz.sha512 create mode 100644 tools/jinja2/LICENSE create mode 100644 tools/jinja2/OWNERS create mode 100644 tools/jinja2/README.chromium create mode 100644 tools/jinja2/__init__.py create mode 100644 tools/jinja2/_compat.py create mode 100644 tools/jinja2/_identifier.py create mode 100644 tools/jinja2/asyncfilters.py create mode 100644 tools/jinja2/asyncsupport.py create mode 100644 tools/jinja2/bccache.py create mode 100644 tools/jinja2/compiler.py create mode 100644 tools/jinja2/constants.py create mode 100644 tools/jinja2/debug.py create mode 100644 tools/jinja2/defaults.py create mode 100644 tools/jinja2/environment.py create mode 100644 tools/jinja2/exceptions.py create mode 100644 tools/jinja2/ext.py create mode 100644 tools/jinja2/filters.py create mode 100755 tools/jinja2/get_jinja2.sh create mode 100644 tools/jinja2/idtracking.py create mode 100644 tools/jinja2/jinja2.gni create mode 100644 tools/jinja2/lexer.py create mode 100644 tools/jinja2/loaders.py create mode 100644 tools/jinja2/meta.py create mode 100644 tools/jinja2/nativetypes.py create mode 100644 tools/jinja2/nodes.py create mode 100644 tools/jinja2/optimizer.py create mode 100644 tools/jinja2/parser.py create mode 100644 tools/jinja2/runtime.py create mode 100644 tools/jinja2/sandbox.py create mode 100644 tools/jinja2/tests.py create mode 100644 tools/jinja2/utils.py create mode 100644 tools/jinja2/visitor.py create mode 100644 tools/markupsafe/AUTHORS create mode 100644 tools/markupsafe/LICENSE create mode 100644 tools/markupsafe/MarkupSafe-0.18.tar.gz.md5 create mode 100644 tools/markupsafe/MarkupSafe-0.18.tar.gz.sha512 create mode 100644 tools/markupsafe/OWNERS create mode 100644 tools/markupsafe/README.chromium create mode 100644 tools/markupsafe/__init__.py create mode 100644 tools/markupsafe/_compat.py create mode 100644 tools/markupsafe/_constants.py create mode 100644 tools/markupsafe/_native.py create mode 100644 tools/markupsafe/_speedups.c create mode 100755 tools/markupsafe/get_markupsafe.sh diff --git a/LICENSE b/LICENSE index 5dffdb791b08..7fc997b4f04d 100644 --- a/LICENSE +++ b/LICENSE @@ -1035,6 +1035,109 @@ The externally maintained libraries used by Node.js are: OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ +- inspector_protocol, located at tools/inspector_protocol, is licensed as follows: + """ + // Copyright 2016 The Chromium Authors. All rights reserved. + // + // Redistribution and use in source and binary forms, with or without + // modification, are permitted provided that the following conditions are + // met: + // + // * Redistributions of source code must retain the above copyright + // notice, this list of conditions and the following disclaimer. + // * Redistributions in binary form must reproduce the above + // copyright notice, this list of conditions and the following disclaimer + // in the documentation and/or other materials provided with the + // distribution. + // * Neither the name of Google Inc. nor the names of its + // contributors may be used to endorse or promote products derived from + // this software without specific prior written permission. + // + // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- jinja2, located at tools/jinja2, is licensed as follows: + """ + Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- markupsafe, located at tools/markupsafe, is licensed as follows: + """ + Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS + for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms of the software as well + as documentation, with or without modification, are permitted provided + that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + """ + - cpplint.py, located at tools/cpplint.py, is licensed as follows: """ Copyright (c) 2009 Google Inc. All rights reserved. diff --git a/node.gyp b/node.gyp index 682111051cb8..8c42474bbed9 100644 --- a/node.gyp +++ b/node.gyp @@ -1045,7 +1045,7 @@ }], # end aix section [ 'v8_enable_inspector==1', { 'variables': { - 'protocol_path': 'deps/v8/third_party/inspector_protocol', + 'protocol_path': 'tools/inspector_protocol', 'node_inspector_path': 'src/inspector', 'node_inspector_generated_sources': [ '<(SHARED_INTERMEDIATE_DIR)/src/node/inspector/protocol/Forward.h', @@ -1105,7 +1105,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConvertProtocolToJSON.py', + 'tools/inspector_protocol/ConvertProtocolToJSON.py', '<@(_inputs)', '<@(_outputs)', ], @@ -1160,7 +1160,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConvertProtocolToJSON.py', + 'tools/inspector_protocol/ConvertProtocolToJSON.py', '<@(_inputs)', '<@(_outputs)', ], @@ -1176,7 +1176,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConcatenateProtocols.py', + 'tools/inspector_protocol/ConcatenateProtocols.py', '<@(_inputs)', '<@(_outputs)', ], diff --git a/tools/inspector_protocol/CheckProtocolCompatibility.py b/tools/inspector_protocol/CheckProtocolCompatibility.py new file mode 100755 index 000000000000..c70162a2a44e --- /dev/null +++ b/tools/inspector_protocol/CheckProtocolCompatibility.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python +# Copyright (c) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Inspector protocol validator. +# +# Tests that subsequent protocol changes are not breaking backwards compatibility. +# Following violations are reported: +# +# - Domain has been removed +# - Command has been removed +# - Required command parameter was added or changed from optional +# - Required response parameter was removed or changed to optional +# - Event has been removed +# - Required event parameter was removed or changed to optional +# - Parameter type has changed. +# +# For the parameters with composite types the above checks are also applied +# recursively to every property of the type. +# +# Adding --show_changes to the command line prints out a list of valid public API changes. + +import copy +import os.path +import optparse +import sys + +try: + import json +except ImportError: + import simplejson as json + + +def list_to_map(items, key): + result = {} + for item in items: + if "experimental" not in item and "hidden" not in item: + result[item[key]] = item + return result + + +def named_list_to_map(container, name, key): + if name in container: + return list_to_map(container[name], key) + return {} + + +def removed(reverse): + if reverse: + return "added" + return "removed" + + +def required(reverse): + if reverse: + return "optional" + return "required" + + +def compare_schemas(d_1, d_2, reverse): + errors = [] + domains_1 = copy.deepcopy(d_1) + domains_2 = copy.deepcopy(d_2) + types_1 = normalize_types_in_schema(domains_1) + types_2 = normalize_types_in_schema(domains_2) + + domains_by_name_1 = list_to_map(domains_1, "domain") + domains_by_name_2 = list_to_map(domains_2, "domain") + + for name in domains_by_name_1: + domain_1 = domains_by_name_1[name] + if name not in domains_by_name_2: + errors.append("%s: domain has been %s" % (name, removed(reverse))) + continue + compare_domains(domain_1, domains_by_name_2[name], types_1, types_2, errors, reverse) + return errors + + +def compare_domains(domain_1, domain_2, types_map_1, types_map_2, errors, reverse): + domain_name = domain_1["domain"] + commands_1 = named_list_to_map(domain_1, "commands", "name") + commands_2 = named_list_to_map(domain_2, "commands", "name") + for name in commands_1: + command_1 = commands_1[name] + if name not in commands_2: + errors.append("%s.%s: command has been %s" % (domain_1["domain"], name, removed(reverse))) + continue + compare_commands(domain_name, command_1, commands_2[name], types_map_1, types_map_2, errors, reverse) + + events_1 = named_list_to_map(domain_1, "events", "name") + events_2 = named_list_to_map(domain_2, "events", "name") + for name in events_1: + event_1 = events_1[name] + if name not in events_2: + errors.append("%s.%s: event has been %s" % (domain_1["domain"], name, removed(reverse))) + continue + compare_events(domain_name, event_1, events_2[name], types_map_1, types_map_2, errors, reverse) + + +def compare_commands(domain_name, command_1, command_2, types_map_1, types_map_2, errors, reverse): + context = domain_name + "." + command_1["name"] + + params_1 = named_list_to_map(command_1, "parameters", "name") + params_2 = named_list_to_map(command_2, "parameters", "name") + # Note the reversed order: we allow removing but forbid adding parameters. + compare_params_list(context, "parameter", params_2, params_1, types_map_2, types_map_1, 0, errors, not reverse) + + returns_1 = named_list_to_map(command_1, "returns", "name") + returns_2 = named_list_to_map(command_2, "returns", "name") + compare_params_list(context, "response parameter", returns_1, returns_2, types_map_1, types_map_2, 0, errors, reverse) + + +def compare_events(domain_name, event_1, event_2, types_map_1, types_map_2, errors, reverse): + context = domain_name + "." + event_1["name"] + params_1 = named_list_to_map(event_1, "parameters", "name") + params_2 = named_list_to_map(event_2, "parameters", "name") + compare_params_list(context, "parameter", params_1, params_2, types_map_1, types_map_2, 0, errors, reverse) + + +def compare_params_list(context, kind, params_1, params_2, types_map_1, types_map_2, depth, errors, reverse): + for name in params_1: + param_1 = params_1[name] + if name not in params_2: + if "optional" not in param_1: + errors.append("%s.%s: required %s has been %s" % (context, name, kind, removed(reverse))) + continue + + param_2 = params_2[name] + if param_2 and "optional" in param_2 and "optional" not in param_1: + errors.append("%s.%s: %s %s is now %s" % (context, name, required(reverse), kind, required(not reverse))) + continue + type_1 = extract_type(param_1, types_map_1, errors) + type_2 = extract_type(param_2, types_map_2, errors) + compare_types(context + "." + name, kind, type_1, type_2, types_map_1, types_map_2, depth, errors, reverse) + + +def compare_types(context, kind, type_1, type_2, types_map_1, types_map_2, depth, errors, reverse): + if depth > 5: + return + + base_type_1 = type_1["type"] + base_type_2 = type_2["type"] + + if base_type_1 != base_type_2: + errors.append("%s: %s base type mismatch, '%s' vs '%s'" % (context, kind, base_type_1, base_type_2)) + elif base_type_1 == "object": + params_1 = named_list_to_map(type_1, "properties", "name") + params_2 = named_list_to_map(type_2, "properties", "name") + # If both parameters have the same named type use it in the context. + if "id" in type_1 and "id" in type_2 and type_1["id"] == type_2["id"]: + type_name = type_1["id"] + else: + type_name = "" + context += " %s->%s" % (kind, type_name) + compare_params_list(context, "property", params_1, params_2, types_map_1, types_map_2, depth + 1, errors, reverse) + elif base_type_1 == "array": + item_type_1 = extract_type(type_1["items"], types_map_1, errors) + item_type_2 = extract_type(type_2["items"], types_map_2, errors) + compare_types(context, kind, item_type_1, item_type_2, types_map_1, types_map_2, depth + 1, errors, reverse) + + +def extract_type(typed_object, types_map, errors): + if "type" in typed_object: + result = {"id": "", "type": typed_object["type"]} + if typed_object["type"] == "object": + result["properties"] = [] + elif typed_object["type"] == "array": + result["items"] = typed_object["items"] + return result + elif "$ref" in typed_object: + ref = typed_object["$ref"] + if ref not in types_map: + errors.append("Can not resolve type: %s" % ref) + types_map[ref] = {"id": "", "type": "object"} + return types_map[ref] + + +def normalize_types_in_schema(domains): + types = {} + for domain in domains: + domain_name = domain["domain"] + normalize_types(domain, domain_name, types) + return types + + +def normalize_types(obj, domain_name, types): + if isinstance(obj, list): + for item in obj: + normalize_types(item, domain_name, types) + elif isinstance(obj, dict): + for key, value in obj.items(): + if key == "$ref" and value.find(".") == -1: + obj[key] = "%s.%s" % (domain_name, value) + elif key == "id": + obj[key] = "%s.%s" % (domain_name, value) + types[obj[key]] = obj + else: + normalize_types(value, domain_name, types) + + +def load_schema(file_name, domains): + # pylint: disable=W0613 + if not os.path.isfile(file_name): + return + input_file = open(file_name, "r") + json_string = input_file.read() + parsed_json = json.loads(json_string) + domains += parsed_json["domains"] + return parsed_json["version"] + + +def self_test(): + def create_test_schema_1(): + return [ + { + "domain": "Network", + "types": [ + { + "id": "LoaderId", + "type": "string" + }, + { + "id": "Headers", + "type": "object" + }, + { + "id": "Request", + "type": "object", + "properties": [ + {"name": "url", "type": "string"}, + {"name": "method", "type": "string"}, + {"name": "headers", "$ref": "Headers"}, + {"name": "becameOptionalField", "type": "string"}, + {"name": "removedField", "type": "string"}, + ] + } + ], + "commands": [ + { + "name": "removedCommand", + }, + { + "name": "setExtraHTTPHeaders", + "parameters": [ + {"name": "headers", "$ref": "Headers"}, + {"name": "mismatched", "type": "string"}, + {"name": "becameOptional", "$ref": "Headers"}, + {"name": "removedRequired", "$ref": "Headers"}, + {"name": "becameRequired", "$ref": "Headers", "optional": True}, + {"name": "removedOptional", "$ref": "Headers", "optional": True}, + ], + "returns": [ + {"name": "mimeType", "type": "string"}, + {"name": "becameOptional", "type": "string"}, + {"name": "removedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string", "optional": True}, + {"name": "removedOptional", "type": "string", "optional": True}, + ] + } + ], + "events": [ + { + "name": "requestWillBeSent", + "parameters": [ + {"name": "frameId", "type": "string", "experimental": True}, + {"name": "request", "$ref": "Request"}, + {"name": "becameOptional", "type": "string"}, + {"name": "removedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string", "optional": True}, + {"name": "removedOptional", "type": "string", "optional": True}, + ] + }, + { + "name": "removedEvent", + "parameters": [ + {"name": "errorText", "type": "string"}, + {"name": "canceled", "type": "boolean", "optional": True} + ] + } + ] + }, + { + "domain": "removedDomain" + } + ] + + def create_test_schema_2(): + return [ + { + "domain": "Network", + "types": [ + { + "id": "LoaderId", + "type": "string" + }, + { + "id": "Request", + "type": "object", + "properties": [ + {"name": "url", "type": "string"}, + {"name": "method", "type": "string"}, + {"name": "headers", "type": "object"}, + {"name": "becameOptionalField", "type": "string", "optional": True}, + ] + } + ], + "commands": [ + { + "name": "addedCommand", + }, + { + "name": "setExtraHTTPHeaders", + "parameters": [ + {"name": "headers", "type": "object"}, + {"name": "mismatched", "type": "object"}, + {"name": "becameOptional", "type": "object", "optional": True}, + {"name": "addedRequired", "type": "object"}, + {"name": "becameRequired", "type": "object"}, + {"name": "addedOptional", "type": "object", "optional": True}, + ], + "returns": [ + {"name": "mimeType", "type": "string"}, + {"name": "becameOptional", "type": "string", "optional": True}, + {"name": "addedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string"}, + {"name": "addedOptional", "type": "string", "optional": True}, + ] + } + ], + "events": [ + { + "name": "requestWillBeSent", + "parameters": [ + {"name": "request", "$ref": "Request"}, + {"name": "becameOptional", "type": "string", "optional": True}, + {"name": "addedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string"}, + {"name": "addedOptional", "type": "string", "optional": True}, + ] + }, + { + "name": "addedEvent" + } + ] + }, + { + "domain": "addedDomain" + } + ] + + expected_errors = [ + "removedDomain: domain has been removed", + "Network.removedCommand: command has been removed", + "Network.removedEvent: event has been removed", + "Network.setExtraHTTPHeaders.mismatched: parameter base type mismatch, 'object' vs 'string'", + "Network.setExtraHTTPHeaders.addedRequired: required parameter has been added", + "Network.setExtraHTTPHeaders.becameRequired: optional parameter is now required", + "Network.setExtraHTTPHeaders.removedRequired: required response parameter has been removed", + "Network.setExtraHTTPHeaders.becameOptional: required response parameter is now optional", + "Network.requestWillBeSent.removedRequired: required parameter has been removed", + "Network.requestWillBeSent.becameOptional: required parameter is now optional", + "Network.requestWillBeSent.request parameter->Network.Request.removedField: required property has been removed", + "Network.requestWillBeSent.request parameter->Network.Request.becameOptionalField: required property is now optional", + ] + + expected_errors_reverse = [ + "addedDomain: domain has been added", + "Network.addedEvent: event has been added", + "Network.addedCommand: command has been added", + "Network.setExtraHTTPHeaders.mismatched: parameter base type mismatch, 'string' vs 'object'", + "Network.setExtraHTTPHeaders.removedRequired: required parameter has been removed", + "Network.setExtraHTTPHeaders.becameOptional: required parameter is now optional", + "Network.setExtraHTTPHeaders.addedRequired: required response parameter has been added", + "Network.setExtraHTTPHeaders.becameRequired: optional response parameter is now required", + "Network.requestWillBeSent.becameRequired: optional parameter is now required", + "Network.requestWillBeSent.addedRequired: required parameter has been added", + ] + + def is_subset(subset, superset, message): + for i in range(len(subset)): + if subset[i] not in superset: + sys.stderr.write("%s error: %s\n" % (message, subset[i])) + return False + return True + + def errors_match(expected, actual): + return (is_subset(actual, expected, "Unexpected") and + is_subset(expected, actual, "Missing")) + + return (errors_match(expected_errors, + compare_schemas(create_test_schema_1(), create_test_schema_2(), False)) and + errors_match(expected_errors_reverse, + compare_schemas(create_test_schema_2(), create_test_schema_1(), True))) + + +def load_domains_and_baselines(file_name, domains, baseline_domains): + version = load_schema(os.path.normpath(file_name), domains) + suffix = "-%s.%s.json" % (version["major"], version["minor"]) + baseline_file = file_name.replace(".json", suffix) + load_schema(os.path.normpath(baseline_file), baseline_domains) + return version + + +def main(): + if not self_test(): + sys.stderr.write("Self-test failed") + return 1 + + cmdline_parser = optparse.OptionParser() + cmdline_parser.add_option("--show_changes") + cmdline_parser.add_option("--expected_errors") + cmdline_parser.add_option("--stamp") + arg_options, arg_values = cmdline_parser.parse_args() + + if len(arg_values) < 1: + sys.stderr.write("Usage: %s [--show_changes] [, ...]\n" % sys.argv[0]) + return 1 + + domains = [] + baseline_domains = [] + version = load_domains_and_baselines(arg_values[0], domains, baseline_domains) + for dependency in arg_values[1:]: + load_domains_and_baselines(dependency, domains, baseline_domains) + + expected_errors = [] + if arg_options.expected_errors: + expected_errors_file = open(arg_options.expected_errors, "r") + expected_errors = json.loads(expected_errors_file.read())["errors"] + expected_errors_file.close() + + errors = compare_schemas(baseline_domains, domains, False) + unexpected_errors = [] + for i in range(len(errors)): + if errors[i] not in expected_errors: + unexpected_errors.append(errors[i]) + if len(unexpected_errors) > 0: + sys.stderr.write(" Compatibility checks FAILED\n") + for error in unexpected_errors: + sys.stderr.write(" %s\n" % error) + return 1 + + if arg_options.show_changes: + changes = compare_schemas(domains, baseline_domains, True) + if len(changes) > 0: + print " Public changes since %s:" % version + for change in changes: + print " %s" % change + + if arg_options.stamp: + with open(arg_options.stamp, 'a') as _: + pass + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/inspector_protocol/CodeGenerator.py b/tools/inspector_protocol/CodeGenerator.py new file mode 100644 index 000000000000..e630b0298571 --- /dev/null +++ b/tools/inspector_protocol/CodeGenerator.py @@ -0,0 +1,654 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import sys +import optparse +import collections +import functools +import re +import copy +try: + import json +except ImportError: + import simplejson as json + +# Path handling for libraries and templates +# Paths have to be normalized because Jinja uses the exact template path to +# determine the hash used in the cache filename, and we need a pre-caching step +# to be concurrency-safe. Use absolute path because __file__ is absolute if +# module is imported, and relative if executed directly. +# If paths differ between pre-caching and individual file compilation, the cache +# is regenerated, which causes a race condition and breaks concurrent build, +# since some compile processes will try to read the partially written cache. +module_path, module_filename = os.path.split(os.path.realpath(__file__)) + +def read_config(): + # pylint: disable=W0703 + def json_to_object(data, output_base, config_base): + def json_object_hook(object_dict): + items = [(k, os.path.join(config_base, v) if k == "path" else v) for (k, v) in object_dict.items()] + items = [(k, os.path.join(output_base, v) if k == "output" else v) for (k, v) in items] + keys, values = zip(*items) + return collections.namedtuple('X', keys)(*values) + return json.loads(data, object_hook=json_object_hook) + + def init_defaults(config_tuple, path, defaults): + keys = list(config_tuple._fields) # pylint: disable=E1101 + values = [getattr(config_tuple, k) for k in keys] + for i in xrange(len(keys)): + if hasattr(values[i], "_fields"): + values[i] = init_defaults(values[i], path + "." + keys[i], defaults) + for optional in defaults: + if optional.find(path + ".") != 0: + continue + optional_key = optional[len(path) + 1:] + if optional_key.find(".") == -1 and optional_key not in keys: + keys.append(optional_key) + values.append(defaults[optional]) + return collections.namedtuple('X', keys)(*values) + + try: + cmdline_parser = optparse.OptionParser() + cmdline_parser.add_option("--output_base") + cmdline_parser.add_option("--jinja_dir") + cmdline_parser.add_option("--config") + cmdline_parser.add_option("--config_value", action="append", type="string") + arg_options, _ = cmdline_parser.parse_args() + jinja_dir = arg_options.jinja_dir + if not jinja_dir: + raise Exception("jinja directory must be specified") + jinja_dir = jinja_dir.decode('utf8') + output_base = arg_options.output_base + if not output_base: + raise Exception("Base output directory must be specified") + output_base = output_base.decode('utf8') + config_file = arg_options.config + if not config_file: + raise Exception("Config file name must be specified") + config_file = config_file.decode('utf8') + config_base = os.path.dirname(config_file) + config_values = arg_options.config_value + if not config_values: + config_values = [] + except Exception: + # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html + exc = sys.exc_info()[1] + sys.stderr.write("Failed to parse command-line arguments: %s\n\n" % exc) + exit(1) + + try: + config_json_file = open(config_file, "r") + config_json_string = config_json_file.read() + config_partial = json_to_object(config_json_string, output_base, config_base) + config_json_file.close() + defaults = { + ".use_snake_file_names": False, + ".use_title_case_methods": False, + ".imported": False, + ".imported.export_macro": "", + ".imported.export_header": False, + ".imported.header": False, + ".imported.package": False, + ".imported.options": False, + ".protocol.export_macro": "", + ".protocol.export_header": False, + ".protocol.options": False, + ".exported": False, + ".exported.export_macro": "", + ".exported.export_header": False, + ".lib": False, + ".lib.export_macro": "", + ".lib.export_header": False, + } + for key_value in config_values: + parts = key_value.split("=") + if len(parts) == 2: + defaults["." + parts[0]] = parts[1] + return (jinja_dir, config_file, init_defaults(config_partial, "", defaults)) + except Exception: + # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html + exc = sys.exc_info()[1] + sys.stderr.write("Failed to parse config file: %s\n\n" % exc) + exit(1) + + +# ---- Begin of utilities exposed to generator ---- + + +def to_title_case(name): + return name[:1].upper() + name[1:] + + +def dash_to_camelcase(word): + prefix = "" + if word[0] == "-": + prefix = "Negative" + word = word[1:] + return prefix + "".join(to_title_case(x) or "-" for x in word.split("-")) + + +def to_snake_case(name): + return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name, sys.maxint).lower() + + +def to_method_case(config, name): + if config.use_title_case_methods: + return to_title_case(name) + return name + + +def join_arrays(dict, keys): + result = [] + for key in keys: + if key in dict: + result += dict[key] + return result + + +def format_include(config, header, file_name=None): + if file_name is not None: + header = header + "/" + file_name + ".h" + header = "\"" + header + "\"" if header[0] not in "<\"" else header + if config.use_snake_file_names: + header = to_snake_case(header) + return header + + +def to_file_name(config, file_name): + if config.use_snake_file_names: + return to_snake_case(file_name).replace(".cpp", ".cc") + return file_name + + +# ---- End of utilities exposed to generator ---- + + +def initialize_jinja_env(jinja_dir, cache_dir, config): + # pylint: disable=F0401 + sys.path.insert(1, os.path.abspath(jinja_dir)) + import jinja2 + + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(module_path), + # Bytecode cache is not concurrency-safe unless pre-cached: + # if pre-cached this is read-only, but writing creates a race condition. + bytecode_cache=jinja2.FileSystemBytecodeCache(cache_dir), + keep_trailing_newline=True, # newline-terminate generated files + lstrip_blocks=True, # so can indent control flow tags + trim_blocks=True) + jinja_env.filters.update({"to_title_case": to_title_case, "dash_to_camelcase": dash_to_camelcase, "to_method_case": functools.partial(to_method_case, config)}) + jinja_env.add_extension("jinja2.ext.loopcontrols") + return jinja_env + + +def create_imported_type_definition(domain_name, type, imported_namespace): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "pass_type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "raw_type": "%s::%s::API::%s" % (imported_namespace, domain_name, type["id"]), + "raw_pass_type": "%s::%s::API::%s*" % (imported_namespace, domain_name, type["id"]), + "raw_return_type": "%s::%s::API::%s*" % (imported_namespace, domain_name, type["id"]), + } + + +def create_user_type_definition(domain_name, type): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr" % (domain_name, type["id"]), + "pass_type": "std::unique_ptr" % (domain_name, type["id"]), + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr" % (domain_name, type["id"]), + "raw_type": "protocol::%s::%s" % (domain_name, type["id"]), + "raw_pass_type": "protocol::%s::%s*" % (domain_name, type["id"]), + "raw_return_type": "protocol::%s::%s*" % (domain_name, type["id"]), + } + + +def create_object_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr", + "pass_type": "std::unique_ptr", + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr", + "raw_type": "protocol::DictionaryValue", + "raw_pass_type": "protocol::DictionaryValue*", + "raw_return_type": "protocol::DictionaryValue*", + } + + +def create_any_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr", + "pass_type": "std::unique_ptr", + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr", + "raw_type": "protocol::Value", + "raw_pass_type": "protocol::Value*", + "raw_return_type": "protocol::Value*", + } + + +def create_string_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "String", + "pass_type": "const String&", + "to_pass_type": "%s", + "to_raw_type": "%s", + "to_rvalue": "%s", + "type": "String", + "raw_type": "String", + "raw_pass_type": "const String&", + "raw_return_type": "String", + } + + +def create_primitive_type_definition(type): + # pylint: disable=W0622 + typedefs = { + "number": "double", + "integer": "int", + "boolean": "bool" + } + defaults = { + "number": "0", + "integer": "0", + "boolean": "false" + } + jsontypes = { + "number": "TypeDouble", + "integer": "TypeInteger", + "boolean": "TypeBoolean", + } + return { + "return_type": typedefs[type], + "pass_type": typedefs[type], + "to_pass_type": "%s", + "to_raw_type": "%s", + "to_rvalue": "%s", + "type": typedefs[type], + "raw_type": typedefs[type], + "raw_pass_type": typedefs[type], + "raw_return_type": typedefs[type], + "default_value": defaults[type] + } + + +def wrap_array_definition(type): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr>" % type["raw_type"], + "pass_type": "std::unique_ptr>" % type["raw_type"], + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr>" % type["raw_type"], + "raw_type": "protocol::Array<%s>" % type["raw_type"], + "raw_pass_type": "protocol::Array<%s>*" % type["raw_type"], + "raw_return_type": "protocol::Array<%s>*" % type["raw_type"], + "out_type": "protocol::Array<%s>&" % type["raw_type"], + } + + +class Protocol(object): + def __init__(self, config): + self.config = config + self.json_api = {"domains": []} + self.imported_domains = [] + self.exported_domains = [] + self.generate_domains = self.read_protocol_file(config.protocol.path) + + if config.protocol.options: + self.generate_domains = [rule.domain for rule in config.protocol.options] + self.exported_domains = [rule.domain for rule in config.protocol.options if hasattr(rule, "exported")] + + if config.imported: + self.imported_domains = self.read_protocol_file(config.imported.path) + if config.imported.options: + self.imported_domains = [rule.domain for rule in config.imported.options] + + self.patch_full_qualified_refs() + self.create_notification_types() + self.create_type_definitions() + self.generate_used_types() + + + def read_protocol_file(self, file_name): + input_file = open(file_name, "r") + json_string = input_file.read() + input_file.close() + parsed_json = json.loads(json_string) + version = parsed_json["version"]["major"] + "." + parsed_json["version"]["minor"] + domains = [] + for domain in parsed_json["domains"]: + domains.append(domain["domain"]) + domain["version"] = version + self.json_api["domains"] += parsed_json["domains"] + return domains + + + def patch_full_qualified_refs(self): + def patch_full_qualified_refs_in_domain(json, domain_name): + if isinstance(json, list): + for item in json: + patch_full_qualified_refs_in_domain(item, domain_name) + if not isinstance(json, dict): + return + for key in json: + if key == "type" and json[key] == "string": + json[key] = domain_name + ".string" + if key != "$ref": + patch_full_qualified_refs_in_domain(json[key], domain_name) + continue + if json["$ref"].find(".") == -1: + json["$ref"] = domain_name + "." + json["$ref"] + return + + for domain in self.json_api["domains"]: + patch_full_qualified_refs_in_domain(domain, domain["domain"]) + + + def all_references(self, json): + refs = set() + if isinstance(json, list): + for item in json: + refs |= self.all_references(item) + if not isinstance(json, dict): + return refs + for key in json: + if key != "$ref": + refs |= self.all_references(json[key]) + else: + refs.add(json["$ref"]) + return refs + + def generate_used_types(self): + all_refs = set() + for domain in self.json_api["domains"]: + domain_name = domain["domain"] + if "commands" in domain: + for command in domain["commands"]: + if self.generate_command(domain_name, command["name"]): + all_refs |= self.all_references(command) + if "events" in domain: + for event in domain["events"]: + if self.generate_event(domain_name, event["name"]): + all_refs |= self.all_references(event) + all_refs.add(domain_name + "." + to_title_case(event["name"]) + "Notification") + + dependencies = self.generate_type_dependencies() + queue = set(all_refs) + while len(queue): + ref = queue.pop() + if ref in dependencies: + queue |= dependencies[ref] - all_refs + all_refs |= dependencies[ref] + self.used_types = all_refs + + + def generate_type_dependencies(self): + dependencies = dict() + domains_with_types = (x for x in self.json_api["domains"] if "types" in x) + for domain in domains_with_types: + domain_name = domain["domain"] + for type in domain["types"]: + related_types = self.all_references(type) + if len(related_types): + dependencies[domain_name + "." + type["id"]] = related_types + return dependencies + + + def create_notification_types(self): + for domain in self.json_api["domains"]: + if "events" in domain: + for event in domain["events"]: + event_type = dict() + event_type["description"] = "Wrapper for notification params" + event_type["type"] = "object" + event_type["id"] = to_title_case(event["name"]) + "Notification" + if "parameters" in event: + event_type["properties"] = copy.deepcopy(event["parameters"]) + if "types" not in domain: + domain["types"] = list() + domain["types"].append(event_type) + + + def create_type_definitions(self): + imported_namespace = "::".join(self.config.imported.namespace) if self.config.imported else "" + self.type_definitions = {} + self.type_definitions["number"] = create_primitive_type_definition("number") + self.type_definitions["integer"] = create_primitive_type_definition("integer") + self.type_definitions["boolean"] = create_primitive_type_definition("boolean") + self.type_definitions["object"] = create_object_type_definition() + self.type_definitions["any"] = create_any_type_definition() + for domain in self.json_api["domains"]: + self.type_definitions[domain["domain"] + ".string"] = create_string_type_definition() + if not ("types" in domain): + continue + for type in domain["types"]: + type_name = domain["domain"] + "." + type["id"] + if type["type"] == "object" and domain["domain"] in self.imported_domains: + self.type_definitions[type_name] = create_imported_type_definition(domain["domain"], type, imported_namespace) + elif type["type"] == "object": + self.type_definitions[type_name] = create_user_type_definition(domain["domain"], type) + elif type["type"] == "array": + items_type = type["items"]["type"] + self.type_definitions[type_name] = wrap_array_definition(self.type_definitions[items_type]) + elif type["type"] == domain["domain"] + ".string": + self.type_definitions[type_name] = create_string_type_definition() + else: + self.type_definitions[type_name] = create_primitive_type_definition(type["type"]) + + + def check_options(self, options, domain, name, include_attr, exclude_attr, default): + for rule in options: + if rule.domain != domain: + continue + if include_attr and hasattr(rule, include_attr): + return name in getattr(rule, include_attr) + if exclude_attr and hasattr(rule, exclude_attr): + return name not in getattr(rule, exclude_attr) + return default + return False + + + # ---- Begin of methods exposed to generator + + + def type_definition(self, name): + return self.type_definitions[name] + + + def resolve_type(self, prop): + if "$ref" in prop: + return self.type_definitions[prop["$ref"]] + if prop["type"] == "array": + return wrap_array_definition(self.resolve_type(prop["items"])) + return self.type_definitions[prop["type"]] + + + def generate_command(self, domain, command): + if not self.config.protocol.options: + return domain in self.generate_domains + return self.check_options(self.config.protocol.options, domain, command, "include", "exclude", True) + + + def generate_event(self, domain, event): + if not self.config.protocol.options: + return domain in self.generate_domains + return self.check_options(self.config.protocol.options, domain, event, "include_events", "exclude_events", True) + + + def generate_type(self, domain, typename): + return domain + "." + typename in self.used_types + + + def is_async_command(self, domain, command): + if not self.config.protocol.options: + return False + return self.check_options(self.config.protocol.options, domain, command, "async", None, False) + + + def is_exported(self, domain, name): + if not self.config.protocol.options: + return False + return self.check_options(self.config.protocol.options, domain, name, "exported", None, False) + + + def is_imported(self, domain, name): + if not self.config.imported: + return False + if not self.config.imported.options: + return domain in self.imported_domains + return self.check_options(self.config.imported.options, domain, name, "imported", None, False) + + + def is_exported_domain(self, domain): + return domain in self.exported_domains + + + def generate_disable(self, domain): + if "commands" not in domain: + return True + for command in domain["commands"]: + if command["name"] == "disable" and self.generate_command(domain["domain"], "disable"): + return False + return True + + + def is_imported_dependency(self, domain): + return domain in self.generate_domains or domain in self.imported_domains + + +def main(): + jinja_dir, config_file, config = read_config() + + protocol = Protocol(config) + + if not config.exported and len(protocol.exported_domains): + sys.stderr.write("Domains [%s] are exported, but config is missing export entry\n\n" % ", ".join(protocol.exported_domains)) + exit(1) + + if not os.path.exists(config.protocol.output): + os.mkdir(config.protocol.output) + if len(protocol.exported_domains) and not os.path.exists(config.exported.output): + os.mkdir(config.exported.output) + jinja_env = initialize_jinja_env(jinja_dir, config.protocol.output, config) + + inputs = [] + inputs.append(__file__) + inputs.append(config_file) + inputs.append(config.protocol.path) + if config.imported: + inputs.append(config.imported.path) + templates_dir = os.path.join(module_path, "templates") + inputs.append(os.path.join(templates_dir, "TypeBuilder_h.template")) + inputs.append(os.path.join(templates_dir, "TypeBuilder_cpp.template")) + inputs.append(os.path.join(templates_dir, "Exported_h.template")) + inputs.append(os.path.join(templates_dir, "Imported_h.template")) + + h_template = jinja_env.get_template("templates/TypeBuilder_h.template") + cpp_template = jinja_env.get_template("templates/TypeBuilder_cpp.template") + exported_template = jinja_env.get_template("templates/Exported_h.template") + imported_template = jinja_env.get_template("templates/Imported_h.template") + + outputs = dict() + + for domain in protocol.json_api["domains"]: + class_name = domain["domain"] + template_context = { + "protocol": protocol, + "config": config, + "domain": domain, + "join_arrays": join_arrays, + "format_include": functools.partial(format_include, config), + } + + if domain["domain"] in protocol.generate_domains: + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".h"))] = h_template.render(template_context) + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".cpp"))] = cpp_template.render(template_context) + if domain["domain"] in protocol.exported_domains: + outputs[os.path.join(config.exported.output, to_file_name(config, class_name + ".h"))] = exported_template.render(template_context) + if domain["domain"] in protocol.imported_domains: + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".h"))] = imported_template.render(template_context) + + if config.lib: + template_context = { + "config": config, + "format_include": functools.partial(format_include, config), + } + + lib_templates_dir = os.path.join(module_path, "lib") + # Note these should be sorted in the right order. + # TODO(dgozman): sort them programmatically based on commented includes. + lib_h_templates = [ + "Collections_h.template", + "ErrorSupport_h.template", + "Values_h.template", + "Object_h.template", + "ValueConversions_h.template", + "Maybe_h.template", + "Array_h.template", + "DispatcherBase_h.template", + "Parser_h.template", + ] + + lib_cpp_templates = [ + "Protocol_cpp.template", + "ErrorSupport_cpp.template", + "Values_cpp.template", + "Object_cpp.template", + "DispatcherBase_cpp.template", + "Parser_cpp.template", + ] + + forward_h_templates = [ + "Forward_h.template", + "Allocator_h.template", + "FrontendChannel_h.template", + ] + + def generate_lib_file(file_name, template_files): + parts = [] + for template_file in template_files: + inputs.append(os.path.join(lib_templates_dir, template_file)) + template = jinja_env.get_template("lib/" + template_file) + parts.append(template.render(template_context)) + outputs[file_name] = "\n\n".join(parts) + + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Forward.h")), forward_h_templates) + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Protocol.h")), lib_h_templates) + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Protocol.cpp")), lib_cpp_templates) + + # Make gyp / make generatos happy, otherwise make rebuilds world. + inputs_ts = max(map(os.path.getmtime, inputs)) + up_to_date = True + for output_file in outputs.iterkeys(): + if not os.path.exists(output_file) or os.path.getmtime(output_file) < inputs_ts: + up_to_date = False + break + if up_to_date: + sys.exit() + + for file_name, content in outputs.iteritems(): + out_file = open(file_name, "w") + out_file.write(content) + out_file.close() + + +main() diff --git a/tools/inspector_protocol/ConcatenateProtocols.py b/tools/inspector_protocol/ConcatenateProtocols.py new file mode 100755 index 000000000000..a7cbc992c76e --- /dev/null +++ b/tools/inspector_protocol/ConcatenateProtocols.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import sys + +try: + import json +except ImportError: + import simplejson as json + + +def main(argv): + if len(argv) < 1: + sys.stderr.write("Usage: %s [ [, ...]] \n" % sys.argv[0]) + return 1 + + domains = [] + version = None + for protocol in argv[:-1]: + file_name = os.path.normpath(protocol) + if not os.path.isfile(file_name): + sys.stderr.write("Cannot find %s\n" % file_name) + return 1 + input_file = open(file_name, "r") + json_string = input_file.read() + parsed_json = json.loads(json_string) + domains += parsed_json["domains"] + version = parsed_json["version"] + + output_file = open(argv[-1], "w") + json.dump({"version": version, "domains": domains}, output_file, indent=4, sort_keys=False, separators=(',', ': ')) + output_file.close() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/inspector_protocol/ConvertProtocolToJSON.py b/tools/inspector_protocol/ConvertProtocolToJSON.py new file mode 100644 index 000000000000..56fc09d78cb1 --- /dev/null +++ b/tools/inspector_protocol/ConvertProtocolToJSON.py @@ -0,0 +1,183 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections +import json +import os.path +import re +import sys + +file_name = None +description = '' + +primitiveTypes = ['integer', 'number', 'boolean', 'string', 'object', 'any', 'array'] + + +def assignType(item, type, isArray=False): + if isArray: + item['type'] = 'array' + item['items'] = collections.OrderedDict() + assignType(item['items'], type) + return + + if type == 'enum': + type = 'string' + if type in primitiveTypes: + item['type'] = type + else: + item['$ref'] = type + + +def createItem(d, experimental, deprecated, name=None): + result = collections.OrderedDict(d) + if name: + result['name'] = name + global description + if description: + result['description'] = description.strip() + if experimental: + result['experimental'] = True + if deprecated: + result['deprecated'] = True + return result + + +def parse(data): + protocol = collections.OrderedDict() + protocol['version'] = collections.OrderedDict() + protocol['domains'] = [] + domain = None + item = None + subitems = None + nukeDescription = False + global description + lines = data.split('\n') + for i in range(0, len(lines)): + if nukeDescription: + description = '' + nukeDescription = False + line = lines[i] + trimLine = line.strip() + + if trimLine.startswith('#'): + if len(description): + description += '\n' + description += trimLine[2:] + continue + else: + nukeDescription = True + + if len(trimLine) == 0: + continue + + match = re.compile('^(experimental )?(deprecated )?domain (.*)').match(line) + if match: + domain = createItem({'domain' : match.group(3)}, match.group(1), match.group(2)) + protocol['domains'].append(domain) + continue + + match = re.compile('^ depends on ([^\s]+)').match(line) + if match: + if 'dependencies' not in domain: + domain['dependencies'] = [] + domain['dependencies'].append(match.group(1)) + continue + + match = re.compile('^ (experimental )?(deprecated )?type (.*) extends (array of )?([^\s]+)').match(line) + if match: + if 'types' not in domain: + domain['types'] = [] + item = createItem({'id': match.group(3)}, match.group(1), match.group(2)) + assignType(item, match.group(5), match.group(4)) + domain['types'].append(item) + continue + + match = re.compile('^ (experimental )?(deprecated )?(command|event) (.*)').match(line) + if match: + list = [] + if match.group(3) == 'command': + if 'commands' in domain: + list = domain['commands'] + else: + list = domain['commands'] = [] + else: + if 'events' in domain: + list = domain['events'] + else: + list = domain['events'] = [] + + item = createItem({}, match.group(1), match.group(2), match.group(4)) + list.append(item) + continue + + match = re.compile('^ (experimental )?(deprecated )?(optional )?(array of )?([^\s]+) ([^\s]+)').match(line) + if match: + param = createItem({}, match.group(1), match.group(2), match.group(6)) + if match.group(3): + param['optional'] = True + assignType(param, match.group(5), match.group(4)) + if match.group(5) == 'enum': + enumliterals = param['enum'] = [] + subitems.append(param) + continue + + match = re.compile('^ (parameters|returns|properties)').match(line) + if match: + subitems = item[match.group(1)] = [] + continue + + match = re.compile('^ enum').match(line) + if match: + enumliterals = item['enum'] = [] + continue + + match = re.compile('^version').match(line) + if match: + continue + + match = re.compile('^ major (\d+)').match(line) + if match: + protocol['version']['major'] = match.group(1) + continue + + match = re.compile('^ minor (\d+)').match(line) + if match: + protocol['version']['minor'] = match.group(1) + continue + + match = re.compile('^ redirect ([^\s]+)').match(line) + if match: + item['redirect'] = match.group(1) + continue + + match = re.compile('^ ( )?[^\s]+$').match(line) + if match: + # enum literal + enumliterals.append(trimLine) + continue + + print 'Error in %s:%s, illegal token: \t%s' % (file_name, i, line) + sys.exit(1) + return protocol + +def main(argv): + if len(argv) < 2: + sys.stderr.write("Usage: %s \n" % sys.argv[0]) + return 1 + global file_name + file_name = os.path.normpath(argv[0]) + input_file = open(file_name, "r") + pdl_string = input_file.read() + protocol = parse(pdl_string) + output_file = open(argv[0].replace('.pdl', '.json'), 'wb') + json.dump(protocol, output_file, indent=4, separators=(',', ': ')) + output_file.close() + + output_file = open(os.path.normpath(argv[1]), 'wb') + json.dump(protocol, output_file, indent=4, separators=(',', ': ')) + output_file.close() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/inspector_protocol/LICENSE b/tools/inspector_protocol/LICENSE new file mode 100644 index 000000000000..800468e57634 --- /dev/null +++ b/tools/inspector_protocol/LICENSE @@ -0,0 +1,27 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/inspector_protocol/OWNERS b/tools/inspector_protocol/OWNERS new file mode 100644 index 000000000000..8d0b6d90cb23 --- /dev/null +++ b/tools/inspector_protocol/OWNERS @@ -0,0 +1,8 @@ +set noparent + +alph@chromium.org +caseq@chromium.org +dgozman@chromium.org +kozyatinskiy@chromium.org +pfeldman@chromium.org +yangguo@chromium.org diff --git a/tools/inspector_protocol/README.v8 b/tools/inspector_protocol/README.v8 new file mode 100644 index 000000000000..8a82f2a9c9d6 --- /dev/null +++ b/tools/inspector_protocol/README.v8 @@ -0,0 +1,16 @@ +Name: inspector protocol +Short Name: inspector_protocol +URL: https://chromium.googlesource.com/deps/inspector_protocol/ +Version: 0 +Revision: 752d4abd13119010cf30e454e8ef9b5fb7ef43a3 +License: BSD +License File: LICENSE +Security Critical: no + +Description: +src/inspector uses these scripts to generate handlers from protocol +description. + +Local modifications: +- This only includes the lib/ and templates/ directories, scripts, build + and the LICENSE files. diff --git a/tools/inspector_protocol/inspector_protocol.gni b/tools/inspector_protocol/inspector_protocol.gni new file mode 100644 index 000000000000..5dcc1f522d86 --- /dev/null +++ b/tools/inspector_protocol/inspector_protocol.gni @@ -0,0 +1,89 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This template will generate inspector protocol source code. The code will +# not be compiled, use get_target_outputs() to compile them. +# +# Inputs +# +# config_file (required) +# Path to json file specifying inspector protocol configuration. +# +# out_dir (required) +# Path to put the generated files in. It must be inside output or +# generated file directory. +# +# outputs (required) +# Files generated. Relative to out_dir. +# +# inputs (optional) +# Extra inputs specified by the config file. +template("inspector_protocol_generate") { + assert(defined(invoker.config_file)) + assert(defined(invoker.out_dir)) + assert(defined(invoker.outputs)) + assert(defined(invoker.inspector_protocol_dir)) + inspector_protocol_dir = invoker.inspector_protocol_dir + + action(target_name) { + script = "$inspector_protocol_dir/CodeGenerator.py" + + inputs = [ + invoker.config_file, + "$inspector_protocol_dir/lib/Allocator_h.template", + "$inspector_protocol_dir/lib/Array_h.template", + "$inspector_protocol_dir/lib/Collections_h.template", + "$inspector_protocol_dir/lib/DispatcherBase_cpp.template", + "$inspector_protocol_dir/lib/DispatcherBase_h.template", + "$inspector_protocol_dir/lib/ErrorSupport_cpp.template", + "$inspector_protocol_dir/lib/ErrorSupport_h.template", + "$inspector_protocol_dir/lib/Forward_h.template", + "$inspector_protocol_dir/lib/FrontendChannel_h.template", + "$inspector_protocol_dir/lib/Maybe_h.template", + "$inspector_protocol_dir/lib/Object_cpp.template", + "$inspector_protocol_dir/lib/Object_h.template", + "$inspector_protocol_dir/lib/Parser_cpp.template", + "$inspector_protocol_dir/lib/Parser_h.template", + "$inspector_protocol_dir/lib/Protocol_cpp.template", + "$inspector_protocol_dir/lib/ValueConversions_h.template", + "$inspector_protocol_dir/lib/Values_cpp.template", + "$inspector_protocol_dir/lib/Values_h.template", + "$inspector_protocol_dir/templates/Exported_h.template", + "$inspector_protocol_dir/templates/Imported_h.template", + "$inspector_protocol_dir/templates/TypeBuilder_cpp.template", + "$inspector_protocol_dir/templates/TypeBuilder_h.template", + ] + if (defined(invoker.inputs)) { + inputs += invoker.inputs + } + + args = [ + "--jinja_dir", + rebase_path("//third_party/", root_build_dir), # jinja is in chromium's third_party + "--output_base", + rebase_path(invoker.out_dir, root_build_dir), + "--config", + rebase_path(invoker.config_file, root_build_dir), + ] + + if (defined(invoker.config_values)) { + foreach(value, invoker.config_values) { + args += [ + "--config_value", + value, + ] + } + } + + outputs = get_path_info(rebase_path(invoker.outputs, ".", invoker.out_dir), + "abspath") + + forward_variables_from(invoker, + [ + "visibility", + "deps", + "public_deps", + ]) + } +} diff --git a/tools/inspector_protocol/inspector_protocol.gypi b/tools/inspector_protocol/inspector_protocol.gypi new file mode 100644 index 000000000000..1fb7119b5fa5 --- /dev/null +++ b/tools/inspector_protocol/inspector_protocol.gypi @@ -0,0 +1,33 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +{ + 'variables': { + 'inspector_protocol_files': [ + 'lib/Allocator_h.template', + 'lib/Array_h.template', + 'lib/Collections_h.template', + 'lib/DispatcherBase_cpp.template', + 'lib/DispatcherBase_h.template', + 'lib/ErrorSupport_cpp.template', + 'lib/ErrorSupport_h.template', + 'lib/Forward_h.template', + 'lib/FrontendChannel_h.template', + 'lib/Maybe_h.template', + 'lib/Object_cpp.template', + 'lib/Object_h.template', + 'lib/Parser_cpp.template', + 'lib/Parser_h.template', + 'lib/Protocol_cpp.template', + 'lib/ValueConversions_h.template', + 'lib/Values_cpp.template', + 'lib/Values_h.template', + 'templates/Exported_h.template', + 'templates/Imported_h.template', + 'templates/TypeBuilder_cpp.template', + 'templates/TypeBuilder_h.template', + 'CodeGenerator.py', + ] + } +} diff --git a/tools/inspector_protocol/lib/Allocator_h.template b/tools/inspector_protocol/lib/Allocator_h.template new file mode 100644 index 000000000000..8f8109d695c5 --- /dev/null +++ b/tools/inspector_protocol/lib/Allocator_h.template @@ -0,0 +1,30 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Allocator_h +#define {{"_".join(config.protocol.namespace)}}_Allocator_h + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +enum NotNullTagEnum { NotNullLiteral }; + +#define PROTOCOL_DISALLOW_NEW() \ + private: \ + void* operator new(size_t) = delete; \ + void* operator new(size_t, NotNullTagEnum, void*) = delete; \ + void* operator new(size_t, void*) = delete; \ + public: + +#define PROTOCOL_DISALLOW_COPY(ClassName) \ + private: \ + ClassName(const ClassName&) = delete; \ + ClassName& operator=(const ClassName&) = delete + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Allocator_h) diff --git a/tools/inspector_protocol/lib/Array_h.template b/tools/inspector_protocol/lib/Array_h.template new file mode 100644 index 000000000000..3854f6e5cd10 --- /dev/null +++ b/tools/inspector_protocol/lib/Array_h.template @@ -0,0 +1,136 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Array_h +#define {{"_".join(config.protocol.namespace)}}_Array_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "ValueConversions.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +class Array { +public: + static std::unique_ptr> create() + { + return std::unique_ptr>(new Array()); + } + + static std::unique_ptr> fromValue(protocol::Value* value, ErrorSupport* errors) + { + protocol::ListValue* array = ListValue::cast(value); + if (!array) { + errors->addError("array expected"); + return nullptr; + } + std::unique_ptr> result(new Array()); + errors->push(); + for (size_t i = 0; i < array->size(); ++i) { + errors->setName(StringUtil::fromInteger(i)); + std::unique_ptr item = ValueConversions::fromValue(array->at(i), errors); + result->m_vector.push_back(std::move(item)); + } + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; + } + + void addItem(std::unique_ptr value) + { + m_vector.push_back(std::move(value)); + } + + size_t length() + { + return m_vector.size(); + } + + T* get(size_t index) + { + return m_vector[index].get(); + } + + std::unique_ptr toValue() + { + std::unique_ptr result = ListValue::create(); + for (auto& item : m_vector) + result->pushValue(ValueConversions::toValue(item)); + return result; + } + +private: + std::vector> m_vector; +}; + +template +class ArrayBase { +public: + static std::unique_ptr> create() + { + return std::unique_ptr>(new Array()); + } + + static std::unique_ptr> fromValue(protocol::Value* value, ErrorSupport* errors) + { + protocol::ListValue* array = ListValue::cast(value); + if (!array) { + errors->addError("array expected"); + return nullptr; + } + errors->push(); + std::unique_ptr> result(new Array()); + for (size_t i = 0; i < array->size(); ++i) { + errors->setName(StringUtil::fromInteger(i)); + T item = ValueConversions::fromValue(array->at(i), errors); + result->m_vector.push_back(item); + } + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; + } + + void addItem(const T& value) + { + m_vector.push_back(value); + } + + size_t length() + { + return m_vector.size(); + } + + T get(size_t index) + { + return m_vector[index]; + } + + std::unique_ptr toValue() + { + std::unique_ptr result = ListValue::create(); + for (auto& item : m_vector) + result->pushValue(ValueConversions::toValue(item)); + return result; + } + +private: + std::vector m_vector; +}; + +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Array_h) diff --git a/tools/inspector_protocol/lib/Collections_h.template b/tools/inspector_protocol/lib/Collections_h.template new file mode 100644 index 000000000000..7505a17bfa6e --- /dev/null +++ b/tools/inspector_protocol/lib/Collections_h.template @@ -0,0 +1,43 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Collections_h +#define {{"_".join(config.protocol.namespace)}}_Collections_h + +#include {{format_include(config.protocol.package, "Forward")}} +#include + +#if defined(__APPLE__) && !defined(_LIBCPP_VERSION) +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template using HashMap = std::map; +template using HashSet = std::set; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#else +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template using HashMap = std::unordered_map; +template using HashSet = std::unordered_set; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // defined(__APPLE__) && !defined(_LIBCPP_VERSION) + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Collections_h) diff --git a/tools/inspector_protocol/lib/DispatcherBase_cpp.template b/tools/inspector_protocol/lib/DispatcherBase_cpp.template new file mode 100644 index 000000000000..cecef743bffc --- /dev/null +++ b/tools/inspector_protocol/lib/DispatcherBase_cpp.template @@ -0,0 +1,354 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "DispatcherBase.h" +//#include "Parser.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +// static +DispatchResponse DispatchResponse::OK() +{ + DispatchResponse result; + result.m_status = kSuccess; + result.m_errorCode = kParseError; + return result; +} + +// static +DispatchResponse DispatchResponse::Error(const String& error) +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kServerError; + result.m_errorMessage = error; + return result; +} + +// static +DispatchResponse DispatchResponse::InternalError() +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kInternalError; + result.m_errorMessage = "Internal error"; + return result; +} + +// static +DispatchResponse DispatchResponse::InvalidParams(const String& error) +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kInvalidParams; + result.m_errorMessage = error; + return result; +} + +// static +DispatchResponse DispatchResponse::FallThrough() +{ + DispatchResponse result; + result.m_status = kFallThrough; + result.m_errorCode = kParseError; + return result; +} + +// static +const char DispatcherBase::kInvalidParamsString[] = "Invalid parameters"; + +DispatcherBase::WeakPtr::WeakPtr(DispatcherBase* dispatcher) : m_dispatcher(dispatcher) { } + +DispatcherBase::WeakPtr::~WeakPtr() +{ + if (m_dispatcher) + m_dispatcher->m_weakPtrs.erase(this); +} + +DispatcherBase::Callback::Callback(std::unique_ptr backendImpl, int callId, int callbackId) + : m_backendImpl(std::move(backendImpl)) + , m_callId(callId) + , m_callbackId(callbackId) { } + +DispatcherBase::Callback::~Callback() = default; + +void DispatcherBase::Callback::dispose() +{ + m_backendImpl = nullptr; +} + +void DispatcherBase::Callback::sendIfActive(std::unique_ptr partialMessage, const DispatchResponse& response) +{ + if (!m_backendImpl || !m_backendImpl->get()) + return; + m_backendImpl->get()->sendResponse(m_callId, response, std::move(partialMessage)); + m_backendImpl = nullptr; +} + +void DispatcherBase::Callback::fallThroughIfActive() +{ + if (!m_backendImpl || !m_backendImpl->get()) + return; + m_backendImpl->get()->markFallThrough(m_callbackId); + m_backendImpl = nullptr; +} + +DispatcherBase::DispatcherBase(FrontendChannel* frontendChannel) + : m_frontendChannel(frontendChannel) + , m_lastCallbackId(0) + , m_lastCallbackFallThrough(false) { } + +DispatcherBase::~DispatcherBase() +{ + clearFrontend(); +} + +int DispatcherBase::nextCallbackId() +{ + m_lastCallbackFallThrough = false; + return ++m_lastCallbackId; +} + +void DispatcherBase::markFallThrough(int callbackId) +{ + DCHECK(callbackId == m_lastCallbackId); + m_lastCallbackFallThrough = true; +} + +void DispatcherBase::sendResponse(int callId, const DispatchResponse& response, std::unique_ptr result) +{ + if (!m_frontendChannel) + return; + if (response.status() == DispatchResponse::kError) { + reportProtocolError(callId, response.errorCode(), response.errorMessage(), nullptr); + return; + } + m_frontendChannel->sendProtocolResponse(callId, InternalResponse::createResponse(callId, std::move(result))); +} + +void DispatcherBase::sendResponse(int callId, const DispatchResponse& response) +{ + sendResponse(callId, response, DictionaryValue::create()); +} + +namespace { + +class ProtocolError : public Serializable { +public: + static std::unique_ptr createErrorResponse(int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) + { + std::unique_ptr protocolError(new ProtocolError(code, errorMessage)); + protocolError->m_callId = callId; + protocolError->m_hasCallId = true; + if (errors && errors->hasErrors()) + protocolError->m_data = errors->errors(); + return protocolError; + } + + static std::unique_ptr createErrorNotification(DispatchResponse::ErrorCode code, const String& errorMessage) + { + return std::unique_ptr(new ProtocolError(code, errorMessage)); + } + + String serialize() override + { + std::unique_ptr error = DictionaryValue::create(); + error->setInteger("code", m_code); + error->setString("message", m_errorMessage); + if (m_data.length()) + error->setString("data", m_data); + std::unique_ptr message = DictionaryValue::create(); + message->setObject("error", std::move(error)); + if (m_hasCallId) + message->setInteger("id", m_callId); + return message->serialize(); + } + + ~ProtocolError() override {} + +private: + ProtocolError(DispatchResponse::ErrorCode code, const String& errorMessage) + : m_code(code) + , m_errorMessage(errorMessage) + { + } + + DispatchResponse::ErrorCode m_code; + String m_errorMessage; + String m_data; + int m_callId = 0; + bool m_hasCallId = false; +}; + +} // namespace + +static void reportProtocolErrorTo(FrontendChannel* frontendChannel, int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) +{ + if (frontendChannel) + frontendChannel->sendProtocolResponse(callId, ProtocolError::createErrorResponse(callId, code, errorMessage, errors)); +} + +static void reportProtocolErrorTo(FrontendChannel* frontendChannel, DispatchResponse::ErrorCode code, const String& errorMessage) +{ + if (frontendChannel) + frontendChannel->sendProtocolNotification(ProtocolError::createErrorNotification(code, errorMessage)); +} + +void DispatcherBase::reportProtocolError(int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) +{ + reportProtocolErrorTo(m_frontendChannel, callId, code, errorMessage, errors); +} + +void DispatcherBase::clearFrontend() +{ + m_frontendChannel = nullptr; + for (auto& weak : m_weakPtrs) + weak->dispose(); + m_weakPtrs.clear(); +} + +std::unique_ptr DispatcherBase::weakPtr() +{ + std::unique_ptr weak(new DispatcherBase::WeakPtr(this)); + m_weakPtrs.insert(weak.get()); + return weak; +} + +UberDispatcher::UberDispatcher(FrontendChannel* frontendChannel) + : m_frontendChannel(frontendChannel) + , m_fallThroughForNotFound(false) { } + +void UberDispatcher::setFallThroughForNotFound(bool fallThroughForNotFound) +{ + m_fallThroughForNotFound = fallThroughForNotFound; +} + +void UberDispatcher::registerBackend(const String& name, std::unique_ptr dispatcher) +{ + m_dispatchers[name] = std::move(dispatcher); +} + +void UberDispatcher::setupRedirects(const HashMap& redirects) +{ + for (const auto& pair : redirects) + m_redirects[pair.first] = pair.second; +} + +DispatchResponse::Status UberDispatcher::dispatch(std::unique_ptr parsedMessage, int* outCallId, String* outMethod) +{ + if (!parsedMessage) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kParseError, "Message must be a valid JSON"); + return DispatchResponse::kError; + } + std::unique_ptr messageObject = DictionaryValue::cast(std::move(parsedMessage)); + if (!messageObject) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must be an object"); + return DispatchResponse::kError; + } + + int callId = 0; + protocol::Value* callIdValue = messageObject->get("id"); + bool success = callIdValue && callIdValue->asInteger(&callId); + if (outCallId) + *outCallId = callId; + if (!success) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must have integer 'id' property"); + return DispatchResponse::kError; + } + + protocol::Value* methodValue = messageObject->get("method"); + String method; + success = methodValue && methodValue->asString(&method); + if (outMethod) + *outMethod = method; + if (!success) { + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kInvalidRequest, "Message must have string 'method' property", nullptr); + return DispatchResponse::kError; + } + + HashMap::iterator redirectIt = m_redirects.find(method); + if (redirectIt != m_redirects.end()) + method = redirectIt->second; + + size_t dotIndex = StringUtil::find(method, "."); + if (dotIndex == StringUtil::kNotFound) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + String domain = StringUtil::substring(method, 0, dotIndex); + auto it = m_dispatchers.find(domain); + if (it == m_dispatchers.end()) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + return it->second->dispatch(callId, method, std::move(messageObject)); +} + +bool UberDispatcher::getCommandName(const String& message, String* method, std::unique_ptr* parsedMessage) +{ + std::unique_ptr value = StringUtil::parseJSON(message); + if (!value) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kParseError, "Message must be a valid JSON"); + return false; + } + + protocol::DictionaryValue* object = DictionaryValue::cast(value.get()); + if (!object) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must be an object"); + return false; + } + + if (!object->getString("method", method)) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must have string 'method' property"); + return false; + } + + parsedMessage->reset(DictionaryValue::cast(value.release())); + return true; +} + +UberDispatcher::~UberDispatcher() = default; + +// static +std::unique_ptr InternalResponse::createResponse(int callId, std::unique_ptr params) +{ + return std::unique_ptr(new InternalResponse(callId, String(), std::move(params))); +} + +// static +std::unique_ptr InternalResponse::createNotification(const String& notification, std::unique_ptr params) +{ + return std::unique_ptr(new InternalResponse(0, notification, std::move(params))); +} + +String InternalResponse::serialize() +{ + std::unique_ptr result = DictionaryValue::create(); + std::unique_ptr params(m_params ? std::move(m_params) : DictionaryValue::create()); + if (m_notification.length()) { + result->setString("method", m_notification); + result->setValue("params", SerializedValue::create(params->serialize())); + } else { + result->setInteger("id", m_callId); + result->setValue("result", SerializedValue::create(params->serialize())); + } + return result->serialize(); +} + +InternalResponse::InternalResponse(int callId, const String& notification, std::unique_ptr params) + : m_callId(callId) + , m_notification(notification) + , m_params(params ? std::move(params) : nullptr) +{ +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/DispatcherBase_h.template b/tools/inspector_protocol/lib/DispatcherBase_h.template new file mode 100644 index 000000000000..d70a4afe71de --- /dev/null +++ b/tools/inspector_protocol/lib/DispatcherBase_h.template @@ -0,0 +1,173 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_DispatcherBase_h +#define {{"_".join(config.protocol.namespace)}}_DispatcherBase_h + +//#include "Collections.h" +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class WeakPtr; + +class {{config.lib.export_macro}} DispatchResponse { +public: + enum Status { + kSuccess = 0, + kError = 1, + kFallThrough = 2, + kAsync = 3 + }; + + enum ErrorCode { + kParseError = -32700, + kInvalidRequest = -32600, + kMethodNotFound = -32601, + kInvalidParams = -32602, + kInternalError = -32603, + kServerError = -32000, + }; + + Status status() const { return m_status; } + const String& errorMessage() const { return m_errorMessage; } + ErrorCode errorCode() const { return m_errorCode; } + bool isSuccess() const { return m_status == kSuccess; } + + static DispatchResponse OK(); + static DispatchResponse Error(const String&); + static DispatchResponse InternalError(); + static DispatchResponse InvalidParams(const String&); + static DispatchResponse FallThrough(); + +private: + Status m_status; + String m_errorMessage; + ErrorCode m_errorCode; +}; + +class {{config.lib.export_macro}} DispatcherBase { + PROTOCOL_DISALLOW_COPY(DispatcherBase); +public: + static const char kInvalidParamsString[]; + class {{config.lib.export_macro}} WeakPtr { + public: + explicit WeakPtr(DispatcherBase*); + ~WeakPtr(); + DispatcherBase* get() { return m_dispatcher; } + void dispose() { m_dispatcher = nullptr; } + + private: + DispatcherBase* m_dispatcher; + }; + + class {{config.lib.export_macro}} Callback { + public: + Callback(std::unique_ptr backendImpl, int callId, int callbackId); + virtual ~Callback(); + void dispose(); + + protected: + void sendIfActive(std::unique_ptr partialMessage, const DispatchResponse& response); + void fallThroughIfActive(); + + private: + std::unique_ptr m_backendImpl; + int m_callId; + int m_callbackId; + }; + + explicit DispatcherBase(FrontendChannel*); + virtual ~DispatcherBase(); + + virtual DispatchResponse::Status dispatch(int callId, const String& method, std::unique_ptr messageObject) = 0; + + void sendResponse(int callId, const DispatchResponse&, std::unique_ptr result); + void sendResponse(int callId, const DispatchResponse&); + + void reportProtocolError(int callId, DispatchResponse::ErrorCode, const String& errorMessage, ErrorSupport* errors); + void clearFrontend(); + + std::unique_ptr weakPtr(); + + int nextCallbackId(); + void markFallThrough(int callbackId); + bool lastCallbackFallThrough() { return m_lastCallbackFallThrough; } + +private: + FrontendChannel* m_frontendChannel; + protocol::HashSet m_weakPtrs; + int m_lastCallbackId; + bool m_lastCallbackFallThrough; +}; + +class {{config.lib.export_macro}} UberDispatcher { + PROTOCOL_DISALLOW_COPY(UberDispatcher); +public: + explicit UberDispatcher(FrontendChannel*); + void registerBackend(const String& name, std::unique_ptr); + void setupRedirects(const HashMap&); + DispatchResponse::Status dispatch(std::unique_ptr message, int* callId = nullptr, String* method = nullptr); + FrontendChannel* channel() { return m_frontendChannel; } + bool fallThroughForNotFound() { return m_fallThroughForNotFound; } + void setFallThroughForNotFound(bool); + bool getCommandName(const String& message, String* method, std::unique_ptr* parsedMessage); + virtual ~UberDispatcher(); + +private: + FrontendChannel* m_frontendChannel; + bool m_fallThroughForNotFound; + HashMap m_redirects; + protocol::HashMap> m_dispatchers; +}; + +class InternalResponse : public Serializable { + PROTOCOL_DISALLOW_COPY(InternalResponse); +public: + static std::unique_ptr createResponse(int callId, std::unique_ptr params); + static std::unique_ptr createNotification(const String& notification, std::unique_ptr params = nullptr); + + String serialize() override; + + ~InternalResponse() override {} + +private: + InternalResponse(int callId, const String& notification, std::unique_ptr params); + + int m_callId; + String m_notification; + std::unique_ptr m_params; +}; + +class InternalRawNotification : public Serializable { +public: + static std::unique_ptr create(const String& notification) + { + return std::unique_ptr(new InternalRawNotification(notification)); + } + ~InternalRawNotification() override {} + + String serialize() override + { + return m_notification; + } + +private: + explicit InternalRawNotification(const String& notification) + : m_notification(notification) + { + } + + String m_notification; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_DispatcherBase_h) diff --git a/tools/inspector_protocol/lib/ErrorSupport_cpp.template b/tools/inspector_protocol/lib/ErrorSupport_cpp.template new file mode 100644 index 000000000000..7b858b8dc48f --- /dev/null +++ b/tools/inspector_protocol/lib/ErrorSupport_cpp.template @@ -0,0 +1,71 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "ErrorSupport.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +ErrorSupport::ErrorSupport() { } +ErrorSupport::~ErrorSupport() { } + +void ErrorSupport::setName(const char* name) +{ + setName(String(name)); +} + +void ErrorSupport::setName(const String& name) +{ + DCHECK(m_path.size()); + m_path[m_path.size() - 1] = name; +} + +void ErrorSupport::push() +{ + m_path.push_back(String()); +} + +void ErrorSupport::pop() +{ + m_path.pop_back(); +} + +void ErrorSupport::addError(const char* error) +{ + addError(String(error)); +} + +void ErrorSupport::addError(const String& error) +{ + StringBuilder builder; + for (size_t i = 0; i < m_path.size(); ++i) { + if (i) + StringUtil::builderAppend(builder, '.'); + StringUtil::builderAppend(builder, m_path[i]); + } + StringUtil::builderAppend(builder, ": "); + StringUtil::builderAppend(builder, error); + m_errors.push_back(StringUtil::builderToString(builder)); +} + +bool ErrorSupport::hasErrors() +{ + return !!m_errors.size(); +} + +String ErrorSupport::errors() +{ + StringBuilder builder; + for (size_t i = 0; i < m_errors.size(); ++i) { + if (i) + StringUtil::builderAppend(builder, "; "); + StringUtil::builderAppend(builder, m_errors[i]); + } + return StringUtil::builderToString(builder); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/ErrorSupport_h.template b/tools/inspector_protocol/lib/ErrorSupport_h.template new file mode 100644 index 000000000000..083f2a5eb0d4 --- /dev/null +++ b/tools/inspector_protocol/lib/ErrorSupport_h.template @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_ErrorSupport_h +#define {{"_".join(config.protocol.namespace)}}_ErrorSupport_h + +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} ErrorSupport { +public: + ErrorSupport(); + ~ErrorSupport(); + + void push(); + void setName(const char*); + void setName(const String&); + void pop(); + void addError(const char*); + void addError(const String&); + bool hasErrors(); + String errors(); + +private: + std::vector m_path; + std::vector m_errors; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_ErrorSupport_h) diff --git a/tools/inspector_protocol/lib/Forward_h.template b/tools/inspector_protocol/lib/Forward_h.template new file mode 100644 index 000000000000..34d1c0d3e946 --- /dev/null +++ b/tools/inspector_protocol/lib/Forward_h.template @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Forward_h +#define {{"_".join(config.protocol.namespace)}}_Forward_h + +{% if config.lib.export_header %} +#include {{format_include(config.lib.export_header)}} +{% endif %} +#include {{format_include(config.lib.string_header)}} + +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template class Array; +class DictionaryValue; +class DispatchResponse; +class ErrorSupport; +class FundamentalValue; +class ListValue; +template class Maybe; +class Object; +using Response = DispatchResponse; +class SerializedValue; +class StringValue; +class UberDispatcher; +class Value; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Forward_h) diff --git a/tools/inspector_protocol/lib/FrontendChannel_h.template b/tools/inspector_protocol/lib/FrontendChannel_h.template new file mode 100644 index 000000000000..0454978b0c8e --- /dev/null +++ b/tools/inspector_protocol/lib/FrontendChannel_h.template @@ -0,0 +1,30 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_FrontendChannel_h +#define {{"_".join(config.protocol.namespace)}}_FrontendChannel_h + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} Serializable { +public: + virtual String serialize() = 0; + virtual ~Serializable() = default; +}; + +class {{config.lib.export_macro}} FrontendChannel { +public: + virtual ~FrontendChannel() { } + virtual void sendProtocolResponse(int callId, std::unique_ptr message) = 0; + virtual void sendProtocolNotification(std::unique_ptr message) = 0; + virtual void flushProtocolNotifications() = 0; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_FrontendChannel_h) diff --git a/tools/inspector_protocol/lib/Maybe_h.template b/tools/inspector_protocol/lib/Maybe_h.template new file mode 100644 index 000000000000..71593acd0e55 --- /dev/null +++ b/tools/inspector_protocol/lib/Maybe_h.template @@ -0,0 +1,86 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Maybe_h +#define {{"_".join(config.protocol.namespace)}}_Maybe_h + +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +class Maybe { +public: + Maybe() : m_value() { } + Maybe(std::unique_ptr value) : m_value(std::move(value)) { } + Maybe(Maybe&& other) : m_value(std::move(other.m_value)) { } + void operator=(std::unique_ptr value) { m_value = std::move(value); } + T* fromJust() const { DCHECK(m_value); return m_value.get(); } + T* fromMaybe(T* defaultValue) const { return m_value ? m_value.get() : defaultValue; } + bool isJust() const { return !!m_value; } + std::unique_ptr takeJust() { DCHECK(m_value); return std::move(m_value); } +private: + std::unique_ptr m_value; +}; + +template +class MaybeBase { +public: + MaybeBase() : m_isJust(false) { } + MaybeBase(T value) : m_isJust(true), m_value(value) { } + MaybeBase(MaybeBase&& other) : m_isJust(other.m_isJust), m_value(std::move(other.m_value)) { } + void operator=(T value) { m_value = value; m_isJust = true; } + T fromJust() const { DCHECK(m_isJust); return m_value; } + T fromMaybe(const T& defaultValue) const { return m_isJust ? m_value : defaultValue; } + bool isJust() const { return m_isJust; } + T takeJust() { DCHECK(m_isJust); return m_value; } + +protected: + bool m_isJust; + T m_value; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(bool value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(int value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(double value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(const String& value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Maybe_h) diff --git a/tools/inspector_protocol/lib/Object_cpp.template b/tools/inspector_protocol/lib/Object_cpp.template new file mode 100644 index 000000000000..91723a71e29c --- /dev/null +++ b/tools/inspector_protocol/lib/Object_cpp.template @@ -0,0 +1,38 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "Object.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +std::unique_ptr Object::fromValue(protocol::Value* value, ErrorSupport* errors) +{ + protocol::DictionaryValue* dictionary = DictionaryValue::cast(value); + if (!dictionary) { + errors->addError("object expected"); + return nullptr; + } + dictionary = static_cast(dictionary->clone().release()); + return std::unique_ptr(new Object(std::unique_ptr(dictionary))); +} + +std::unique_ptr Object::toValue() const +{ + return DictionaryValue::cast(m_object->clone()); +} + +std::unique_ptr Object::clone() const +{ + return std::unique_ptr(new Object(DictionaryValue::cast(m_object->clone()))); +} + +Object::Object(std::unique_ptr object) : m_object(std::move(object)) { } + +Object::~Object() { } + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Object_h.template b/tools/inspector_protocol/lib/Object_h.template new file mode 100644 index 000000000000..f6ffc57659e1 --- /dev/null +++ b/tools/inspector_protocol/lib/Object_h.template @@ -0,0 +1,32 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Object_h +#define {{"_".join(config.protocol.namespace)}}_Object_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} Object { +public: + static std::unique_ptr fromValue(protocol::Value*, ErrorSupport*); + ~Object(); + + std::unique_ptr toValue() const; + std::unique_ptr clone() const; +private: + explicit Object(std::unique_ptr); + std::unique_ptr m_object; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Object_h) diff --git a/tools/inspector_protocol/lib/Parser_cpp.template b/tools/inspector_protocol/lib/Parser_cpp.template new file mode 100644 index 000000000000..f3dde5ac218e --- /dev/null +++ b/tools/inspector_protocol/lib/Parser_cpp.template @@ -0,0 +1,547 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +namespace { + +const int stackLimit = 1000; + +enum Token { + ObjectBegin, + ObjectEnd, + ArrayBegin, + ArrayEnd, + StringLiteral, + Number, + BoolTrue, + BoolFalse, + NullToken, + ListSeparator, + ObjectPairSeparator, + InvalidToken, +}; + +const char* const nullString = "null"; +const char* const trueString = "true"; +const char* const falseString = "false"; + +bool isASCII(uint16_t c) +{ + return !(c & ~0x7F); +} + +bool isSpaceOrNewLine(uint16_t c) +{ + return isASCII(c) && c <= ' ' && (c == ' ' || (c <= 0xD && c >= 0x9)); +} + +double charactersToDouble(const uint16_t* characters, size_t length, bool* ok) +{ + std::vector buffer; + buffer.reserve(length + 1); + for (size_t i = 0; i < length; ++i) { + if (!isASCII(characters[i])) { + *ok = false; + return 0; + } + buffer.push_back(static_cast(characters[i])); + } + buffer.push_back('\0'); + return StringUtil::toDouble(buffer.data(), length, ok); +} + +double charactersToDouble(const uint8_t* characters, size_t length, bool* ok) +{ + std::string buffer(reinterpret_cast(characters), length); + return StringUtil::toDouble(buffer.data(), length, ok); +} + +template +bool parseConstToken(const Char* start, const Char* end, const Char** tokenEnd, const char* token) +{ + while (start < end && *token != '\0' && *start++ == *token++) { } + if (*token != '\0') + return false; + *tokenEnd = start; + return true; +} + +template +bool readInt(const Char* start, const Char* end, const Char** tokenEnd, bool canHaveLeadingZeros) +{ + if (start == end) + return false; + bool haveLeadingZero = '0' == *start; + int length = 0; + while (start < end && '0' <= *start && *start <= '9') { + ++start; + ++length; + } + if (!length) + return false; + if (!canHaveLeadingZeros && length > 1 && haveLeadingZero) + return false; + *tokenEnd = start; + return true; +} + +template +bool parseNumberToken(const Char* start, const Char* end, const Char** tokenEnd) +{ + // We just grab the number here. We validate the size in DecodeNumber. + // According to RFC4627, a valid number is: [minus] int [frac] [exp] + if (start == end) + return false; + Char c = *start; + if ('-' == c) + ++start; + + if (!readInt(start, end, &start, false)) + return false; + if (start == end) { + *tokenEnd = start; + return true; + } + + // Optional fraction part + c = *start; + if ('.' == c) { + ++start; + if (!readInt(start, end, &start, true)) + return false; + if (start == end) { + *tokenEnd = start; + return true; + } + c = *start; + } + + // Optional exponent part + if ('e' == c || 'E' == c) { + ++start; + if (start == end) + return false; + c = *start; + if ('-' == c || '+' == c) { + ++start; + if (start == end) + return false; + } + if (!readInt(start, end, &start, true)) + return false; + } + + *tokenEnd = start; + return true; +} + +template +bool readHexDigits(const Char* start, const Char* end, const Char** tokenEnd, int digits) +{ + if (end - start < digits) + return false; + for (int i = 0; i < digits; ++i) { + Char c = *start++; + if (!(('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F'))) + return false; + } + *tokenEnd = start; + return true; +} + +template +bool parseStringToken(const Char* start, const Char* end, const Char** tokenEnd) +{ + while (start < end) { + Char c = *start++; + if ('\\' == c) { + if (start == end) + return false; + c = *start++; + // Make sure the escaped char is valid. + switch (c) { + case 'x': + if (!readHexDigits(start, end, &start, 2)) + return false; + break; + case 'u': + if (!readHexDigits(start, end, &start, 4)) + return false; + break; + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + break; + default: + return false; + } + } else if ('"' == c) { + *tokenEnd = start; + return true; + } + } + return false; +} + +template +bool skipComment(const Char* start, const Char* end, const Char** commentEnd) +{ + if (start == end) + return false; + + if (*start != '/' || start + 1 >= end) + return false; + ++start; + + if (*start == '/') { + // Single line comment, read to newline. + for (++start; start < end; ++start) { + if (*start == '\n' || *start == '\r') { + *commentEnd = start + 1; + return true; + } + } + *commentEnd = end; + // Comment reaches end-of-input, which is fine. + return true; + } + + if (*start == '*') { + Char previous = '\0'; + // Block comment, read until end marker. + for (++start; start < end; previous = *start++) { + if (previous == '*' && *start == '/') { + *commentEnd = start + 1; + return true; + } + } + // Block comment must close before end-of-input. + return false; + } + + return false; +} + +template +void skipWhitespaceAndComments(const Char* start, const Char* end, const Char** whitespaceEnd) +{ + while (start < end) { + if (isSpaceOrNewLine(*start)) { + ++start; + } else if (*start == '/') { + const Char* commentEnd; + if (!skipComment(start, end, &commentEnd)) + break; + start = commentEnd; + } else { + break; + } + } + *whitespaceEnd = start; +} + +template +Token parseToken(const Char* start, const Char* end, const Char** tokenStart, const Char** tokenEnd) +{ + skipWhitespaceAndComments(start, end, tokenStart); + start = *tokenStart; + + if (start == end) + return InvalidToken; + + switch (*start) { + case 'n': + if (parseConstToken(start, end, tokenEnd, nullString)) + return NullToken; + break; + case 't': + if (parseConstToken(start, end, tokenEnd, trueString)) + return BoolTrue; + break; + case 'f': + if (parseConstToken(start, end, tokenEnd, falseString)) + return BoolFalse; + break; + case '[': + *tokenEnd = start + 1; + return ArrayBegin; + case ']': + *tokenEnd = start + 1; + return ArrayEnd; + case ',': + *tokenEnd = start + 1; + return ListSeparator; + case '{': + *tokenEnd = start + 1; + return ObjectBegin; + case '}': + *tokenEnd = start + 1; + return ObjectEnd; + case ':': + *tokenEnd = start + 1; + return ObjectPairSeparator; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (parseNumberToken(start, end, tokenEnd)) + return Number; + break; + case '"': + if (parseStringToken(start + 1, end, tokenEnd)) + return StringLiteral; + break; + } + return InvalidToken; +} + +template +int hexToInt(Char c) +{ + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + DCHECK(false); + return 0; +} + +template +bool decodeString(const Char* start, const Char* end, StringBuilder* output) +{ + while (start < end) { + uint16_t c = *start++; + if ('\\' != c) { + StringUtil::builderAppend(*output, c); + continue; + } + if (start == end) + return false; + c = *start++; + + if (c == 'x') { + // \x is not supported. + return false; + } + + switch (c) { + case '"': + case '/': + case '\\': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'v': + c = '\v'; + break; + case 'u': + c = (hexToInt(*start) << 12) + + (hexToInt(*(start + 1)) << 8) + + (hexToInt(*(start + 2)) << 4) + + hexToInt(*(start + 3)); + start += 4; + break; + default: + return false; + } + StringUtil::builderAppend(*output, c); + } + return true; +} + +template +bool decodeString(const Char* start, const Char* end, String* output) +{ + if (start == end) { + *output = ""; + return true; + } + if (start > end) + return false; + StringBuilder buffer; + StringUtil::builderReserve(buffer, end - start); + if (!decodeString(start, end, &buffer)) + return false; + *output = StringUtil::builderToString(buffer); + return true; +} + +template +std::unique_ptr buildValue(const Char* start, const Char* end, const Char** valueTokenEnd, int depth) +{ + if (depth > stackLimit) + return nullptr; + + std::unique_ptr result; + const Char* tokenStart; + const Char* tokenEnd; + Token token = parseToken(start, end, &tokenStart, &tokenEnd); + switch (token) { + case InvalidToken: + return nullptr; + case NullToken: + result = Value::null(); + break; + case BoolTrue: + result = FundamentalValue::create(true); + break; + case BoolFalse: + result = FundamentalValue::create(false); + break; + case Number: { + bool ok; + double value = charactersToDouble(tokenStart, tokenEnd - tokenStart, &ok); + if (!ok) + return nullptr; + int number = static_cast(value); + if (number == value) + result = FundamentalValue::create(number); + else + result = FundamentalValue::create(value); + break; + } + case StringLiteral: { + String value; + bool ok = decodeString(tokenStart + 1, tokenEnd - 1, &value); + if (!ok) + return nullptr; + result = StringValue::create(value); + break; + } + case ArrayBegin: { + std::unique_ptr array = ListValue::create(); + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + while (token != ArrayEnd) { + std::unique_ptr arrayNode = buildValue(start, end, &tokenEnd, depth + 1); + if (!arrayNode) + return nullptr; + array->pushValue(std::move(arrayNode)); + + // After a list value, we expect a comma or the end of the list. + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ListSeparator) { + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ArrayEnd) + return nullptr; + } else if (token != ArrayEnd) { + // Unexpected value after list value. Bail out. + return nullptr; + } + } + if (token != ArrayEnd) + return nullptr; + result = std::move(array); + break; + } + case ObjectBegin: { + std::unique_ptr object = DictionaryValue::create(); + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + while (token != ObjectEnd) { + if (token != StringLiteral) + return nullptr; + String key; + if (!decodeString(tokenStart + 1, tokenEnd - 1, &key)) + return nullptr; + start = tokenEnd; + + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token != ObjectPairSeparator) + return nullptr; + start = tokenEnd; + + std::unique_ptr value = buildValue(start, end, &tokenEnd, depth + 1); + if (!value) + return nullptr; + object->setValue(key, std::move(value)); + start = tokenEnd; + + // After a key/value pair, we expect a comma or the end of the + // object. + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ListSeparator) { + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ObjectEnd) + return nullptr; + } else if (token != ObjectEnd) { + // Unexpected value after last object value. Bail out. + return nullptr; + } + } + if (token != ObjectEnd) + return nullptr; + result = std::move(object); + break; + } + + default: + // We got a token that's not a value. + return nullptr; + } + + skipWhitespaceAndComments(tokenEnd, end, valueTokenEnd); + return result; +} + +template +std::unique_ptr parseJSONInternal(const Char* start, unsigned length) +{ + const Char* end = start + length; + const Char *tokenEnd; + std::unique_ptr value = buildValue(start, end, &tokenEnd, 0); + if (!value || tokenEnd != end) + return nullptr; + return value; +} + +} // anonymous namespace + +std::unique_ptr parseJSONCharacters(const uint16_t* characters, unsigned length) +{ + return parseJSONInternal(characters, length); +} + +std::unique_ptr parseJSONCharacters(const uint8_t* characters, unsigned length) +{ + return parseJSONInternal(characters, length); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Parser_h.template b/tools/inspector_protocol/lib/Parser_h.template new file mode 100644 index 000000000000..8397d3f5d691 --- /dev/null +++ b/tools/inspector_protocol/lib/Parser_h.template @@ -0,0 +1,22 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Parser_h +#define {{"_".join(config.protocol.namespace)}}_Parser_h + +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +{{config.lib.export_macro}} std::unique_ptr parseJSONCharacters(const uint8_t*, unsigned); +{{config.lib.export_macro}} std::unique_ptr parseJSONCharacters(const uint16_t*, unsigned); + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Parser_h) diff --git a/tools/inspector_protocol/lib/Protocol_cpp.template b/tools/inspector_protocol/lib/Protocol_cpp.template new file mode 100644 index 000000000000..901656373a4f --- /dev/null +++ b/tools/inspector_protocol/lib/Protocol_cpp.template @@ -0,0 +1,12 @@ +// This file is generated. + +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include {{format_include(config.protocol.package, "Protocol")}} + +#include +#include + +#include diff --git a/tools/inspector_protocol/lib/ValueConversions_h.template b/tools/inspector_protocol/lib/ValueConversions_h.template new file mode 100644 index 000000000000..4d64ec9091a6 --- /dev/null +++ b/tools/inspector_protocol/lib/ValueConversions_h.template @@ -0,0 +1,171 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_ValueConversions_h +#define {{"_".join(config.protocol.namespace)}}_ValueConversions_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + return T::fromValue(value, errors); + } + + static std::unique_ptr toValue(T* value) + { + return value->toValue(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->toValue(); + } +}; + +template<> +struct ValueConversions { + static bool fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool result = false; + bool success = value ? value->asBoolean(&result) : false; + if (!success) + errors->addError("boolean value expected"); + return result; + } + + static std::unique_ptr toValue(bool value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static int fromValue(protocol::Value* value, ErrorSupport* errors) + { + int result = 0; + bool success = value ? value->asInteger(&result) : false; + if (!success) + errors->addError("integer value expected"); + return result; + } + + static std::unique_ptr toValue(int value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static double fromValue(protocol::Value* value, ErrorSupport* errors) + { + double result = 0; + bool success = value ? value->asDouble(&result) : false; + if (!success) + errors->addError("double value expected"); + return result; + } + + static std::unique_ptr toValue(double value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static String fromValue(protocol::Value* value, ErrorSupport* errors) + { + String result; + bool success = value ? value->asString(&result) : false; + if (!success) + errors->addError("string value expected"); + return result; + } + + static std::unique_ptr toValue(const String& value) + { + return StringValue::create(value); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = !!value; + if (!success) { + errors->addError("value expected"); + return nullptr; + } + return value->clone(); + } + + static std::unique_ptr toValue(Value* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = value && value->type() == protocol::Value::TypeObject; + if (!success) + errors->addError("object expected"); + return DictionaryValue::cast(value->clone()); + } + + static std::unique_ptr toValue(DictionaryValue* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = value && value->type() == protocol::Value::TypeArray; + if (!success) + errors->addError("list expected"); + return ListValue::cast(value->clone()); + } + + static std::unique_ptr toValue(ListValue* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_ValueConversions_h) diff --git a/tools/inspector_protocol/lib/Values_cpp.template b/tools/inspector_protocol/lib/Values_cpp.template new file mode 100644 index 000000000000..b9f061346bf6 --- /dev/null +++ b/tools/inspector_protocol/lib/Values_cpp.template @@ -0,0 +1,409 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +namespace { + +const char* const nullValueString = "null"; +const char* const trueValueString = "true"; +const char* const falseValueString = "false"; + +inline bool escapeChar(uint16_t c, StringBuilder* dst) +{ + switch (c) { + case '\b': StringUtil::builderAppend(*dst, "\\b"); break; + case '\f': StringUtil::builderAppend(*dst, "\\f"); break; + case '\n': StringUtil::builderAppend(*dst, "\\n"); break; + case '\r': StringUtil::builderAppend(*dst, "\\r"); break; + case '\t': StringUtil::builderAppend(*dst, "\\t"); break; + case '\\': StringUtil::builderAppend(*dst, "\\\\"); break; + case '"': StringUtil::builderAppend(*dst, "\\\""); break; + default: + return false; + } + return true; +} + +const char hexDigits[17] = "0123456789ABCDEF"; + +void appendUnsignedAsHex(uint16_t number, StringBuilder* dst) +{ + StringUtil::builderAppend(*dst, "\\u"); + for (size_t i = 0; i < 4; ++i) { + uint16_t c = hexDigits[(number & 0xF000) >> 12]; + StringUtil::builderAppend(*dst, c); + number <<= 4; + } +} + +template +void escapeStringForJSONInternal(const Char* str, unsigned len, + StringBuilder* dst) +{ + for (unsigned i = 0; i < len; ++i) { + Char c = str[i]; + if (escapeChar(c, dst)) + continue; + if (c < 32 || c > 126) { + appendUnsignedAsHex(c, dst); + } else { + StringUtil::builderAppend(*dst, c); + } + } +} + +} // anonymous namespace + +bool Value::asBoolean(bool*) const +{ + return false; +} + +bool Value::asDouble(double*) const +{ + return false; +} + +bool Value::asInteger(int*) const +{ + return false; +} + +bool Value::asString(String*) const +{ + return false; +} + +bool Value::asSerialized(String*) const +{ + return false; +} + +void Value::writeJSON(StringBuilder* output) const +{ + DCHECK(m_type == TypeNull); + StringUtil::builderAppend(*output, nullValueString, 4); +} + +std::unique_ptr Value::clone() const +{ + return Value::null(); +} + +String Value::serialize() +{ + StringBuilder result; + StringUtil::builderReserve(result, 512); + writeJSON(&result); + return StringUtil::builderToString(result); +} + +bool FundamentalValue::asBoolean(bool* output) const +{ + if (type() != TypeBoolean) + return false; + *output = m_boolValue; + return true; +} + +bool FundamentalValue::asDouble(double* output) const +{ + if (type() == TypeDouble) { + *output = m_doubleValue; + return true; + } + if (type() == TypeInteger) { + *output = m_integerValue; + return true; + } + return false; +} + +bool FundamentalValue::asInteger(int* output) const +{ + if (type() != TypeInteger) + return false; + *output = m_integerValue; + return true; +} + +void FundamentalValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeBoolean || type() == TypeInteger || type() == TypeDouble); + if (type() == TypeBoolean) { + if (m_boolValue) + StringUtil::builderAppend(*output, trueValueString, 4); + else + StringUtil::builderAppend(*output, falseValueString, 5); + } else if (type() == TypeDouble) { + if (!std::isfinite(m_doubleValue)) { + StringUtil::builderAppend(*output, nullValueString, 4); + return; + } + StringUtil::builderAppend(*output, StringUtil::fromDouble(m_doubleValue)); + } else if (type() == TypeInteger) { + StringUtil::builderAppend(*output, StringUtil::fromInteger(m_integerValue)); + } +} + +std::unique_ptr FundamentalValue::clone() const +{ + switch (type()) { + case TypeDouble: return FundamentalValue::create(m_doubleValue); + case TypeInteger: return FundamentalValue::create(m_integerValue); + case TypeBoolean: return FundamentalValue::create(m_boolValue); + default: + DCHECK(false); + } + return nullptr; +} + +bool StringValue::asString(String* output) const +{ + *output = m_stringValue; + return true; +} + +void StringValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeString); + StringUtil::builderAppendQuotedString(*output, m_stringValue); +} + +std::unique_ptr StringValue::clone() const +{ + return StringValue::create(m_stringValue); +} + +bool SerializedValue::asSerialized(String* output) const +{ + *output = m_serializedValue; + return true; +} + +void SerializedValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeSerialized); + StringUtil::builderAppend(*output, m_serializedValue); +} + +std::unique_ptr SerializedValue::clone() const +{ + return SerializedValue::create(m_serializedValue); +} + +DictionaryValue::~DictionaryValue() +{ +} + +void DictionaryValue::setBoolean(const String& name, bool value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setInteger(const String& name, int value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setDouble(const String& name, double value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setString(const String& name, const String& value) +{ + setValue(name, StringValue::create(value)); +} + +void DictionaryValue::setValue(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +void DictionaryValue::setObject(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +void DictionaryValue::setArray(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +bool DictionaryValue::getBoolean(const String& name, bool* output) const +{ + protocol::Value* value = get(name); + if (!value) + return false; + return value->asBoolean(output); +} + +bool DictionaryValue::getInteger(const String& name, int* output) const +{ + Value* value = get(name); + if (!value) + return false; + return value->asInteger(output); +} + +bool DictionaryValue::getDouble(const String& name, double* output) const +{ + Value* value = get(name); + if (!value) + return false; + return value->asDouble(output); +} + +bool DictionaryValue::getString(const String& name, String* output) const +{ + protocol::Value* value = get(name); + if (!value) + return false; + return value->asString(output); +} + +DictionaryValue* DictionaryValue::getObject(const String& name) const +{ + return DictionaryValue::cast(get(name)); +} + +protocol::ListValue* DictionaryValue::getArray(const String& name) const +{ + return ListValue::cast(get(name)); +} + +protocol::Value* DictionaryValue::get(const String& name) const +{ + Dictionary::const_iterator it = m_data.find(name); + if (it == m_data.end()) + return nullptr; + return it->second.get(); +} + +DictionaryValue::Entry DictionaryValue::at(size_t index) const +{ + const String key = m_order[index]; + return std::make_pair(key, m_data.find(key)->second.get()); +} + +bool DictionaryValue::booleanProperty(const String& name, bool defaultValue) const +{ + bool result = defaultValue; + getBoolean(name, &result); + return result; +} + +int DictionaryValue::integerProperty(const String& name, int defaultValue) const +{ + int result = defaultValue; + getInteger(name, &result); + return result; +} + +double DictionaryValue::doubleProperty(const String& name, double defaultValue) const +{ + double result = defaultValue; + getDouble(name, &result); + return result; +} + +void DictionaryValue::remove(const String& name) +{ + m_data.erase(name); + m_order.erase(std::remove(m_order.begin(), m_order.end(), name), m_order.end()); +} + +void DictionaryValue::writeJSON(StringBuilder* output) const +{ + StringUtil::builderAppend(*output, '{'); + for (size_t i = 0; i < m_order.size(); ++i) { + Dictionary::const_iterator it = m_data.find(m_order[i]); + CHECK(it != m_data.end()); + if (i) + StringUtil::builderAppend(*output, ','); + StringUtil::builderAppendQuotedString(*output, it->first); + StringUtil::builderAppend(*output, ':'); + it->second->writeJSON(output); + } + StringUtil::builderAppend(*output, '}'); +} + +std::unique_ptr DictionaryValue::clone() const +{ + std::unique_ptr result = DictionaryValue::create(); + for (size_t i = 0; i < m_order.size(); ++i) { + String key = m_order[i]; + Dictionary::const_iterator value = m_data.find(key); + DCHECK(value != m_data.cend() && value->second); + result->setValue(key, value->second->clone()); + } + return std::move(result); +} + +DictionaryValue::DictionaryValue() + : Value(TypeObject) +{ +} + +ListValue::~ListValue() +{ +} + +void ListValue::writeJSON(StringBuilder* output) const +{ + StringUtil::builderAppend(*output, '['); + bool first = true; + for (const std::unique_ptr& value : m_data) { + if (!first) + StringUtil::builderAppend(*output, ','); + value->writeJSON(output); + first = false; + } + StringUtil::builderAppend(*output, ']'); +} + +std::unique_ptr ListValue::clone() const +{ + std::unique_ptr result = ListValue::create(); + for (const std::unique_ptr& value : m_data) + result->pushValue(value->clone()); + return std::move(result); +} + +ListValue::ListValue() + : Value(TypeArray) +{ +} + +void ListValue::pushValue(std::unique_ptr value) +{ + DCHECK(value); + m_data.push_back(std::move(value)); +} + +protocol::Value* ListValue::at(size_t index) +{ + DCHECK_LT(index, m_data.size()); + return m_data[index].get(); +} + +void escapeLatinStringForJSON(const uint8_t* str, unsigned len, StringBuilder* dst) +{ + escapeStringForJSONInternal(str, len, dst); +} + +void escapeWideStringForJSON(const uint16_t* str, unsigned len, StringBuilder* dst) +{ + escapeStringForJSONInternal(str, len, dst); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Values_h.template b/tools/inspector_protocol/lib/Values_h.template new file mode 100644 index 000000000000..3638b34b4e77 --- /dev/null +++ b/tools/inspector_protocol/lib/Values_h.template @@ -0,0 +1,249 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Values_h +#define {{"_".join(config.protocol.namespace)}}_Values_h + +//#include "Allocator.h" +//#include "Collections.h" +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class ListValue; +class DictionaryValue; +class Value; + +class {{config.lib.export_macro}} Value : public Serializable { + PROTOCOL_DISALLOW_COPY(Value); +public: + virtual ~Value() override { } + + static std::unique_ptr null() + { + return std::unique_ptr(new Value()); + } + + enum ValueType { + TypeNull = 0, + TypeBoolean, + TypeInteger, + TypeDouble, + TypeString, + TypeObject, + TypeArray, + TypeSerialized + }; + + ValueType type() const { return m_type; } + + bool isNull() const { return m_type == TypeNull; } + + virtual bool asBoolean(bool* output) const; + virtual bool asDouble(double* output) const; + virtual bool asInteger(int* output) const; + virtual bool asString(String* output) const; + virtual bool asSerialized(String* output) const; + + virtual void writeJSON(StringBuilder* output) const; + virtual std::unique_ptr clone() const; + String serialize() override; + +protected: + Value() : m_type(TypeNull) { } + explicit Value(ValueType type) : m_type(type) { } + +private: + friend class DictionaryValue; + friend class ListValue; + + ValueType m_type; +}; + +class {{config.lib.export_macro}} FundamentalValue : public Value { +public: + static std::unique_ptr create(bool value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + static std::unique_ptr create(int value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + static std::unique_ptr create(double value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + bool asBoolean(bool* output) const override; + bool asDouble(double* output) const override; + bool asInteger(int* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit FundamentalValue(bool value) : Value(TypeBoolean), m_boolValue(value) { } + explicit FundamentalValue(int value) : Value(TypeInteger), m_integerValue(value) { } + explicit FundamentalValue(double value) : Value(TypeDouble), m_doubleValue(value) { } + + union { + bool m_boolValue; + double m_doubleValue; + int m_integerValue; + }; +}; + +class {{config.lib.export_macro}} StringValue : public Value { +public: + static std::unique_ptr create(const String& value) + { + return std::unique_ptr(new StringValue(value)); + } + + static std::unique_ptr create(const char* value) + { + return std::unique_ptr(new StringValue(value)); + } + + bool asString(String* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit StringValue(const String& value) : Value(TypeString), m_stringValue(value) { } + explicit StringValue(const char* value) : Value(TypeString), m_stringValue(value) { } + + String m_stringValue; +}; + +class {{config.lib.export_macro}} SerializedValue : public Value { +public: + static std::unique_ptr create(const String& value) + { + return std::unique_ptr(new SerializedValue(value)); + } + + bool asSerialized(String* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit SerializedValue(const String& value) : Value(TypeSerialized), m_serializedValue(value) { } + + String m_serializedValue; +}; + +class {{config.lib.export_macro}} DictionaryValue : public Value { +public: + using Entry = std::pair; + static std::unique_ptr create() + { + return std::unique_ptr(new DictionaryValue()); + } + + static DictionaryValue* cast(Value* value) + { + if (!value || value->type() != TypeObject) + return nullptr; + return static_cast(value); + } + + static std::unique_ptr cast(std::unique_ptr value) + { + return std::unique_ptr(DictionaryValue::cast(value.release())); + } + + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + + size_t size() const { return m_data.size(); } + + void setBoolean(const String& name, bool); + void setInteger(const String& name, int); + void setDouble(const String& name, double); + void setString(const String& name, const String&); + void setValue(const String& name, std::unique_ptr); + void setObject(const String& name, std::unique_ptr); + void setArray(const String& name, std::unique_ptr); + + bool getBoolean(const String& name, bool* output) const; + bool getInteger(const String& name, int* output) const; + bool getDouble(const String& name, double* output) const; + bool getString(const String& name, String* output) const; + + DictionaryValue* getObject(const String& name) const; + ListValue* getArray(const String& name) const; + Value* get(const String& name) const; + Entry at(size_t index) const; + + bool booleanProperty(const String& name, bool defaultValue) const; + int integerProperty(const String& name, int defaultValue) const; + double doubleProperty(const String& name, double defaultValue) const; + void remove(const String& name); + + ~DictionaryValue() override; + +private: + DictionaryValue(); + template + void set(const String& key, std::unique_ptr& value) + { + DCHECK(value); + bool isNew = m_data.find(key) == m_data.end(); + m_data[key] = std::move(value); + if (isNew) + m_order.push_back(key); + } + + using Dictionary = protocol::HashMap>; + Dictionary m_data; + std::vector m_order; +}; + +class {{config.lib.export_macro}} ListValue : public Value { +public: + static std::unique_ptr create() + { + return std::unique_ptr(new ListValue()); + } + + static ListValue* cast(Value* value) + { + if (!value || value->type() != TypeArray) + return nullptr; + return static_cast(value); + } + + static std::unique_ptr cast(std::unique_ptr value) + { + return std::unique_ptr(ListValue::cast(value.release())); + } + + ~ListValue() override; + + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + + void pushValue(std::unique_ptr); + + Value* at(size_t index); + size_t size() const { return m_data.size(); } + +private: + ListValue(); + std::vector> m_data; +}; + +void escapeLatinStringForJSON(const uint8_t* str, unsigned len, StringBuilder* dst); +void escapeWideStringForJSON(const uint16_t* str, unsigned len, StringBuilder* dst); + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // {{"_".join(config.protocol.namespace)}}_Values_h diff --git a/tools/inspector_protocol/templates/Exported_h.template b/tools/inspector_protocol/templates/Exported_h.template new file mode 100644 index 000000000000..3d36ecffae3c --- /dev/null +++ b/tools/inspector_protocol/templates/Exported_h.template @@ -0,0 +1,65 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h + +{% if config.exported.export_header %} +#include {{format_include(config.exported.export_header)}} +{% endif %} +#include {{format_include(config.exported.string_header)}} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { +namespace API { + +// ------------- Enums. + {% for type in domain.types %} + {% if ("enum" in type) and protocol.is_exported(domain.domain, type.id) %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +{{config.exported.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // {{type.id}}Enum + {% endif %} + {% endfor %} + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if ("enum" in param) and protocol.is_exported(domain.domain, command.name + "." + param.name) %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +{{config.exported.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // {{param.name | to_title_case}}Enum +} // {{command.name | to_title_case }} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Types. + {% for type in domain.types %} + {% if not (type.type == "object") or not ("properties" in type) or not protocol.is_exported(domain.domain, type.id) %}{% continue %}{% endif %} + +class {{config.exported.export_macro}} {{type.id}} { +public: + virtual {{config.exported.string_out}} toJSONString() const = 0; + virtual ~{{type.id}}() { } + static std::unique_ptr fromJSONString(const {{config.exported.string_in}}& json); +}; + {% endfor %} + +} // namespace API +} // namespace {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h) diff --git a/tools/inspector_protocol/templates/Imported_h.template b/tools/inspector_protocol/templates/Imported_h.template new file mode 100644 index 000000000000..4c9d24bd5fcc --- /dev/null +++ b/tools/inspector_protocol/templates/Imported_h.template @@ -0,0 +1,55 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h + +#include {{format_include(config.protocol.package, "Protocol")}} +{% if config.imported.header %} +#include {{format_include(config.imported.header)}} +{% else %} +#include {{format_include(config.imported.package, domain.domain)}} +{% endif %} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + {% for type in domain.types %} + {% if not (type.type == "object") or not ("properties" in type) or not protocol.is_imported(domain.domain, type.id) %}{% continue %}{% endif %} + +template<> +struct ValueConversions<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}> { + static std::unique_ptr<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}> fromValue(protocol::Value* value, ErrorSupport* errors) + { + if (!value) { + errors->addError("value expected"); + return nullptr; + } + String json = value->serialize(); + auto result = {{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}::fromJSONString({{config.imported.to_imported_string % "json"}}); + if (!result) + errors->addError("cannot parse"); + return result; + } + + static std::unique_ptr toValue(const {{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}* value) + { + auto json = value->toJSONString(); + return SerializedValue::create({{config.imported.from_imported_string % "std::move(json)"}}); + } + + static std::unique_ptr toValue(const std::unique_ptr<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}>& value) + { + return toValue(value.get()); + } +}; + {% endfor %} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h) diff --git a/tools/inspector_protocol/templates/TypeBuilder_cpp.template b/tools/inspector_protocol/templates/TypeBuilder_cpp.template new file mode 100644 index 000000000000..026c1cdb8da9 --- /dev/null +++ b/tools/inspector_protocol/templates/TypeBuilder_cpp.template @@ -0,0 +1,397 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include {{format_include(config.protocol.package, domain.domain)}} + +#include {{format_include(config.protocol.package, "Protocol")}} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { + +// ------------- Enum values from types. + +const char Metainfo::domainName[] = "{{domain.domain}}"; +const char Metainfo::commandPrefix[] = "{{domain.domain}}."; +const char Metainfo::version[] = "{{domain.version}}"; + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %} {% endif %} + {% if "enum" in type %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +const char* {{ literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} +} // namespace {{type.id}}Enum + {% if protocol.is_exported(domain.domain, type.id) %} + +namespace API { +namespace {{type.id}}Enum { + {% for literal in type.enum %} +const char* {{ literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} +} // namespace {{type.id}}Enum +} // namespace API + {% endif %} + {% endif %} + {% for property in type.properties %} + {% if "enum" in property %} + + {% for literal in property.enum %} +const char* {{type.id}}::{{property.name | to_title_case}}Enum::{{literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} + {% endif %} + {% endfor %} + {% if not (type.type == "object") or not ("properties" in type) %}{% continue %}{% endif %} + +std::unique_ptr<{{type.id}}> {{type.id}}::fromValue(protocol::Value* value, ErrorSupport* errors) +{ + if (!value || value->type() != protocol::Value::TypeObject) { + errors->addError("object expected"); + return nullptr; + } + + std::unique_ptr<{{type.id}}> result(new {{type.id}}()); + protocol::DictionaryValue* object = DictionaryValue::cast(value); + errors->push(); + {% for property in type.properties %} + protocol::Value* {{property.name}}Value = object->get("{{property.name}}"); + {% if property.optional %} + if ({{property.name}}Value) { + errors->setName("{{property.name}}"); + result->m_{{property.name}} = ValueConversions<{{protocol.resolve_type(property).raw_type}}>::fromValue({{property.name}}Value, errors); + } + {% else %} + errors->setName("{{property.name}}"); + result->m_{{property.name}} = ValueConversions<{{protocol.resolve_type(property).raw_type}}>::fromValue({{property.name}}Value, errors); + {% endif %} + {% endfor %} + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; +} + +std::unique_ptr {{type.id}}::toValue() const +{ + std::unique_ptr result = DictionaryValue::create(); + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_field = "m_" + property.name %} + {% if property.optional %} + if ({{property_field}}.isJust()) + result->setValue("{{property.name}}", ValueConversions<{{property_type.raw_type}}>::toValue({{property_field}}.fromJust())); + {% else %} + result->setValue("{{property.name}}", ValueConversions<{{property_type.raw_type}}>::toValue({{property_type.to_raw_type % property_field}})); + {% endif %} + {% endfor %} + return result; +} + +std::unique_ptr<{{type.id}}> {{type.id}}::clone() const +{ + ErrorSupport errors; + return fromValue(toValue().get(), &errors); +} + {% if protocol.is_exported(domain.domain, type.id) %} + +{{config.exported.string_out}} {{type.id}}::toJSONString() const +{ + String json = toValue()->serialize(); + return {{config.exported.to_string_out % "json"}}; +} + +// static +std::unique_ptr API::{{type.id}}::fromJSONString(const {{config.exported.string_in}}& json) +{ + ErrorSupport errors; + std::unique_ptr value = StringUtil::parseJSON(json); + if (!value) + return nullptr; + return protocol::{{domain.domain}}::{{type.id}}::fromValue(value.get(), &errors); +} + {% endif %} + {% endfor %} + +// ------------- Enum values from params. + + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if "enum" in param %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +const char* {{ literal | to_title_case}} = "{{literal}}"; + {% endfor %} +} // namespace {{param.name | to_title_case}}Enum +} // namespace {{command.name | to_title_case }} + {% if protocol.is_exported(domain.domain, command.name + "." + param.name) %} + +namespace API { +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +const char* {{ literal | to_title_case}} = "{{literal}}"; + {% endfor %} +} // namespace {{param.name | to_title_case}}Enum +} // namespace {{command.name | to_title_case }} +} // namespace API + {% endif %} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Frontend notifications. + {% for event in domain.events %} + {% if not protocol.generate_event(domain.domain, event.name) %}{% continue %}{% endif %} + +void Frontend::{{event.name | to_method_case}}( + {%- for parameter in event.parameters %} + {% if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} + {%- endif %} {{parameter.name}}{%- if not loop.last -%}, {% endif -%} + {% endfor -%}) +{ + if (!m_frontendChannel) + return; + {% if event.parameters %} + std::unique_ptr<{{event.name | to_title_case}}Notification> messageData = {{event.name | to_title_case}}Notification::{{"create" | to_method_case}}() + {% for parameter in event.parameters %} + {% if not "optional" in parameter %} + .{{"set" | to_method_case}}{{parameter.name | to_title_case}}({{protocol.resolve_type(parameter).to_pass_type % parameter.name}}) + {% endif %} + {% endfor %} + .{{ "build" | to_method_case }}(); + {% for parameter in event.parameters %} + {% if "optional" in parameter %} + if ({{parameter.name}}.isJust()) + messageData->{{"set" | to_method_case}}{{parameter.name | to_title_case}}(std::move({{parameter.name}}).takeJust()); + {% endif %} + {% endfor %} + m_frontendChannel->sendProtocolNotification(InternalResponse::createNotification("{{domain.domain}}.{{event.name}}", std::move(messageData))); + {% else %} + m_frontendChannel->sendProtocolNotification(InternalResponse::createNotification("{{domain.domain}}.{{event.name}}")); + {% endif %} +} + {% endfor %} + +void Frontend::flush() +{ + m_frontendChannel->flushProtocolNotifications(); +} + +void Frontend::sendRawNotification(const String& notification) +{ + m_frontendChannel->sendProtocolNotification(InternalRawNotification::create(notification)); +} + +// --------------------- Dispatcher. + +class DispatcherImpl : public protocol::DispatcherBase { +public: + DispatcherImpl(FrontendChannel* frontendChannel, Backend* backend, bool fallThroughForNotFound) + : DispatcherBase(frontendChannel) + , m_backend(backend) + , m_fallThroughForNotFound(fallThroughForNotFound) { + {% for command in domain.commands %} + {% if "redirect" in command %} + m_redirects["{{domain.domain}}.{{command.name}}"] = "{{command.redirect}}.{{command.name}}"; + {% continue %} + {% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + m_dispatchMap["{{domain.domain}}.{{command.name}}"] = &DispatcherImpl::{{command.name}}; + {% endfor %} + } + ~DispatcherImpl() override { } + DispatchResponse::Status dispatch(int callId, const String& method, std::unique_ptr messageObject) override; + HashMap& redirects() { return m_redirects; } + +protected: + using CallHandler = DispatchResponse::Status (DispatcherImpl::*)(int callId, std::unique_ptr messageObject, ErrorSupport* errors); + using DispatchMap = protocol::HashMap; + DispatchMap m_dispatchMap; + HashMap m_redirects; + + {% for command in domain.commands %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + DispatchResponse::Status {{command.name}}(int callId, std::unique_ptr requestMessageObject, ErrorSupport*); + {% endfor %} + + Backend* m_backend; + bool m_fallThroughForNotFound; +}; + +DispatchResponse::Status DispatcherImpl::dispatch(int callId, const String& method, std::unique_ptr messageObject) +{ + protocol::HashMap::iterator it = m_dispatchMap.find(method); + if (it == m_dispatchMap.end()) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolError(callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + + protocol::ErrorSupport errors; + return (this->*(it->second))(callId, std::move(messageObject), &errors); +} + + {% for command in domain.commands %} + {% set command_name_title = command.name | to_title_case %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + {% if protocol.is_async_command(domain.domain, command.name) %} + +class {{command_name_title}}CallbackImpl : public Backend::{{command_name_title}}Callback, public DispatcherBase::Callback { +public: + {{command_name_title}}CallbackImpl(std::unique_ptr backendImpl, int callId, int callbackId) + : DispatcherBase::Callback(std::move(backendImpl), callId, callbackId) { } + + void sendSuccess( + {%- for parameter in command.returns -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%} + {%- if not loop.last -%}, {% endif -%} + {%- endfor -%}) override + { + std::unique_ptr resultObject = DictionaryValue::create(); + {% for parameter in command.returns %} + {% if "optional" in parameter %} + if ({{parameter.name}}.isJust()) + resultObject->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{parameter.name}}.fromJust())); + {% else %} + resultObject->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{protocol.resolve_type(parameter).to_raw_type % parameter.name}})); + {% endif %} + {% endfor %} + sendIfActive(std::move(resultObject), DispatchResponse::OK()); + } + + void fallThrough() override + { + fallThroughIfActive(); + } + + void sendFailure(const DispatchResponse& response) override + { + DCHECK(response.status() == DispatchResponse::kError); + sendIfActive(nullptr, response); + } +}; + {% endif %} + +DispatchResponse::Status DispatcherImpl::{{command.name}}(int callId, std::unique_ptr requestMessageObject, ErrorSupport* errors) +{ + {% if "parameters" in command %} + // Prepare input parameters. + protocol::DictionaryValue* object = DictionaryValue::cast(requestMessageObject->get("params")); + errors->push(); + {% for parameter in command.parameters %} + {% set parameter_type = protocol.resolve_type(parameter) %} + protocol::Value* {{parameter.name}}Value = object ? object->get("{{parameter.name}}") : nullptr; + {% if parameter.optional %} + Maybe<{{parameter_type.raw_type}}> in_{{parameter.name}}; + if ({{parameter.name}}Value) { + errors->setName("{{parameter.name}}"); + in_{{parameter.name}} = ValueConversions<{{parameter_type.raw_type}}>::fromValue({{parameter.name}}Value, errors); + } + {% else %} + errors->setName("{{parameter.name}}"); + {{parameter_type.type}} in_{{parameter.name}} = ValueConversions<{{parameter_type.raw_type}}>::fromValue({{parameter.name}}Value, errors); + {% endif %} + {% endfor %} + errors->pop(); + if (errors->hasErrors()) { + reportProtocolError(callId, DispatchResponse::kInvalidParams, kInvalidParamsString, errors); + return DispatchResponse::kError; + } + {% endif %} + {% if "returns" in command and not protocol.is_async_command(domain.domain, command.name) %} + // Declare output parameters. + {% for parameter in command.returns %} + {% if "optional" in parameter %} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> out_{{parameter.name}}; + {% else %} + {{protocol.resolve_type(parameter).type}} out_{{parameter.name}}; + {% endif %} + {% endfor %} + {% endif %} + + {% if not protocol.is_async_command(domain.domain, command.name) %} + std::unique_ptr weak = weakPtr(); + DispatchResponse response = m_backend->{{command.name | to_method_case}}( + {%- for parameter in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in parameter -%} + std::move(in_{{parameter.name}}) + {%- else -%} + {{protocol.resolve_type(parameter).to_pass_type % ("in_" + parameter.name)}} + {%- endif -%} + {%- endfor %} + {%- if "returns" in command %} + {%- for parameter in command.returns -%} + {%- if not loop.first or command.parameters -%}, {% endif -%} + &out_{{parameter.name}} + {%- endfor %} + {% endif %}); + if (response.status() == DispatchResponse::kFallThrough) + return response.status(); + {% if "returns" in command %} + std::unique_ptr result = DictionaryValue::create(); + if (response.status() == DispatchResponse::kSuccess) { + {% for parameter in command.returns %} + {% if "optional" in parameter %} + if (out_{{parameter.name}}.isJust()) + result->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue(out_{{parameter.name}}.fromJust())); + {% else %} + result->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{protocol.resolve_type(parameter).to_raw_type % ("out_" + parameter.name)}})); + {% endif %} + {% endfor %} + } + if (weak->get()) + weak->get()->sendResponse(callId, response, std::move(result)); + {% else %} + if (weak->get()) + weak->get()->sendResponse(callId, response); + {% endif %} + return response.status(); + {% else %} + std::unique_ptr weak = weakPtr(); + std::unique_ptr<{{command_name_title}}CallbackImpl> callback(new {{command.name | to_title_case}}CallbackImpl(weakPtr(), callId, nextCallbackId())); + m_backend->{{command.name | to_method_case}}( + {%- for property in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in property -%} + std::move(in_{{property.name}}) + {%- else -%} + {{protocol.resolve_type(property).to_pass_type % ("in_" + property.name)}} + {%- endif -%} + {%- endfor -%} + {%- if command.parameters -%}, {% endif -%} + std::move(callback)); + return (weak->get() && weak->get()->lastCallbackFallThrough()) ? DispatchResponse::kFallThrough : DispatchResponse::kAsync; + {% endif %} +} + {% endfor %} + +// static +void Dispatcher::wire(UberDispatcher* uber, Backend* backend) +{ + std::unique_ptr dispatcher(new DispatcherImpl(uber->channel(), backend, uber->fallThroughForNotFound())); + uber->setupRedirects(dispatcher->redirects()); + uber->registerBackend("{{domain.domain}}", std::move(dispatcher)); +} + +} // {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/templates/TypeBuilder_h.template b/tools/inspector_protocol/templates/TypeBuilder_h.template new file mode 100644 index 000000000000..744d496026a2 --- /dev/null +++ b/tools/inspector_protocol/templates/TypeBuilder_h.template @@ -0,0 +1,301 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h + +{% if config.protocol.export_header %} +#include {{format_include(config.protocol.export_header)}} +{% endif %} +#include {{format_include(config.protocol.package, "Protocol")}} +// For each imported domain we generate a ValueConversions struct instead of a full domain definition +// and include Domain::API version from there. +{% for name in domain.dependencies %} + {% if protocol.is_imported_dependency(name) %} +#include {{format_include(config.protocol.package, name)}} + {% endif %} +{% endfor %} +{% if protocol.is_exported_domain(domain.domain) %} +#include {{format_include(config.exported.package, domain.domain)}} +{% endif %} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { + +// ------------- Forward and enum declarations. + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if type.type == "object" %} + {% if "properties" in type %} +class {{type.id}}; + {% else %} +using {{type.id}} = Object; + {% endif %} + {% elif type.type != "array" %} +using {{type.id}} = {{protocol.resolve_type(type).type}}; + {% endif %} + {% endfor %} + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if "enum" in type %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +{{config.protocol.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // namespace {{type.id}}Enum + {% endif %} + {% endfor %} + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if "enum" in param %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +{{config.protocol.export_macro}} extern const char* {{literal | dash_to_camelcase}}; + {% endfor %} +} // {{param.name | to_title_case}}Enum +} // {{command.name | to_title_case }} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Type and builder declarations. + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if not (type.type == "object") or not ("properties" in type) %}{% continue %}{% endif %} + +class {{config.protocol.export_macro}} {{type.id}} : public Serializable{% if protocol.is_exported(domain.domain, type.id) %}, public API::{{type.id}}{% endif %}{ + PROTOCOL_DISALLOW_COPY({{type.id}}); +public: + static std::unique_ptr<{{type.id}}> fromValue(protocol::Value* value, ErrorSupport* errors); + + ~{{type.id}}() override { } + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_name = property.name | to_title_case %} + {% set property_field = "m_" + property.name %} + {% if "enum" in property %} + + struct {{config.protocol.export_macro}} {{property_name}}Enum { + {% for literal in property.enum %} + static const char* {{literal | dash_to_camelcase}}; + {% endfor %} + }; // {{property_name}}Enum + {% endif %} + + {% if property.optional %} + bool {{"has" | to_method_case}}{{property_name}}() { return {{property_field}}.isJust(); } + {{property_type.raw_return_type}} {{"get" | to_method_case}}{{property_name}}({{property_type.raw_pass_type}} defaultValue) { return {{property_field}}.isJust() ? {{property_field}}.fromJust() : defaultValue; } + {% else %} + {{property_type.raw_return_type}} {{"get" | to_method_case}}{{property_name}}() { return {{property_type.to_raw_type % property_field}}; } + {% endif %} + void {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) { {{property_field}} = {{property_type.to_rvalue % "value"}}; } + {% endfor %} + + std::unique_ptr toValue() const; + String serialize() override { return toValue()->serialize(); } + std::unique_ptr<{{type.id}}> clone() const; + {% if protocol.is_exported(domain.domain, type.id) %} + {{config.exported.string_out}} toJSONString() const override; + {% endif %} + + template + class {{type.id}}Builder { + public: + enum { + NoFieldsSet = 0, + {% for property in type.properties|rejectattr("optional") %} + {{property.name | to_title_case}}Set = 1 << {{loop.index}}, + {% endfor %} + AllFieldsSet = ( + {%- for property in type.properties %} + {% if not(property.optional) %}{{property.name | to_title_case}}Set | {%endif %} + {% endfor %}0)}; + + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_name = property.name | to_title_case %} + + {% if property.optional %} + {{type.id}}Builder& {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) + { + m_result->{{"set" | to_method_case}}{{property_name}}({{property_type.to_rvalue % "value"}}); + return *this; + } + {% else %} + {{type.id}}Builder& {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) + { + static_assert(!(STATE & {{property_name}}Set), "property {{property.name}} should not be set yet"); + m_result->{{"set" | to_method_case}}{{property_name}}({{property_type.to_rvalue % "value"}}); + return castState<{{property_name}}Set>(); + } + {% endif %} + {% endfor %} + + std::unique_ptr<{{type.id}}> {{"build" | to_method_case}}() + { + static_assert(STATE == AllFieldsSet, "state should be AllFieldsSet"); + return std::move(m_result); + } + + private: + friend class {{type.id}}; + {{type.id}}Builder() : m_result(new {{type.id}}()) { } + + template {{type.id}}Builder& castState() + { + return *reinterpret_cast<{{type.id}}Builder*>(this); + } + + {{protocol.type_definition(domain.domain + "." + type.id).type}} m_result; + }; + + static {{type.id}}Builder<0> {{"create" | to_method_case}}() + { + return {{type.id}}Builder<0>(); + } + +private: + {{type.id}}() + { + {% for property in type.properties %} + {% if not(property.optional) and "default_value" in protocol.resolve_type(property) %} + m_{{property.name}} = {{protocol.resolve_type(property).default_value}}; + {%endif %} + {% endfor %} + } + + {% for property in type.properties %} + {% if property.optional %} + Maybe<{{protocol.resolve_type(property).raw_type}}> m_{{property.name}}; + {% else %} + {{protocol.resolve_type(property).type}} m_{{property.name}}; + {% endif %} + {% endfor %} +}; + + {% endfor %} + +// ------------- Backend interface. + +class {{config.protocol.export_macro}} Backend { +public: + virtual ~Backend() { } + + {% for command in domain.commands %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + {% if protocol.is_async_command(domain.domain, command.name) %} + class {{config.protocol.export_macro}} {{command.name | to_title_case}}Callback { + public: + virtual void sendSuccess( + {%- for parameter in command.returns -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%} + {%- if not loop.last -%}, {% endif -%} + {%- endfor -%} + ) = 0; + virtual void sendFailure(const DispatchResponse&) = 0; + virtual void fallThrough() = 0; + virtual ~{{command.name | to_title_case}}Callback() { } + }; + {% endif %} + {%- if not protocol.is_async_command(domain.domain, command.name) %} + virtual DispatchResponse {{command.name | to_method_case}}( + {%- else %} + virtual void {{command.name | to_method_case}}( + {%- endif %} + {%- for parameter in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> in_{{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} in_{{parameter.name}} + {%- endif -%} + {%- endfor -%} + {%- if protocol.is_async_command(domain.domain, command.name) -%} + {%- if command.parameters -%}, {% endif -%} + std::unique_ptr<{{command.name | to_title_case}}Callback> callback + {%- else -%} + {%- for parameter in command.returns -%} + {%- if (not loop.first) or command.parameters -%}, {% endif -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}>* out_{{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).type}}* out_{{parameter.name}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + ) = 0; + {% endfor %} + + {% if protocol.generate_disable(domain) %} + virtual DispatchResponse {{"disable" | to_method_case}}() + { + return DispatchResponse::OK(); + } + {% endif %} +}; + +// ------------- Frontend interface. + +class {{config.protocol.export_macro}} Frontend { +public: + explicit Frontend(FrontendChannel* frontendChannel) : m_frontendChannel(frontendChannel) { } + {% for event in domain.events %} + {% if not protocol.generate_event(domain.domain, event.name) %}{% continue %}{% endif %} + void {{event.name | to_method_case}}( + {%- for parameter in event.parameters -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} = Maybe<{{protocol.resolve_type(parameter).raw_type}}>() + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%}{%- if not loop.last -%}, {% endif -%} + {%- endfor -%} + ); + {% endfor %} + + void flush(); + void sendRawNotification(const String&); +private: + FrontendChannel* m_frontendChannel; +}; + +// ------------- Dispatcher. + +class {{config.protocol.export_macro}} Dispatcher { +public: + static void wire(UberDispatcher*, Backend*); + +private: + Dispatcher() { } +}; + +// ------------- Metainfo. + +class {{config.protocol.export_macro}} Metainfo { +public: + using BackendClass = Backend; + using FrontendClass = Frontend; + using DispatcherClass = Dispatcher; + static const char domainName[]; + static const char commandPrefix[]; + static const char version[]; +}; + +} // namespace {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h) diff --git a/tools/jinja2/AUTHORS b/tools/jinja2/AUTHORS new file mode 100644 index 000000000000..fd6dbfcbfd8f --- /dev/null +++ b/tools/jinja2/AUTHORS @@ -0,0 +1,34 @@ +Jinja is written and maintained by the Jinja Team and various +contributors: + +Lead Developer: + +- Armin Ronacher + +Developers: + +- Christoph Hack +- Georg Brandl + +Contributors: + +- Bryan McLemore +- Mickaël Guérin +- Cameron Knight +- Lawrence Journal-World. +- David Cramer +- Adrian Mönnich (ThiefMaster) + +Patches and suggestions: + +- Ronny Pfannschmidt +- Axel Böhm +- Alexey Melchakov +- Bryan McLemore +- Clovis Fabricio (nosklo) +- Cameron Knight +- Peter van Dijk (Habbie) +- Stefan Ebner +- Rene Leonhardt +- Thomas Waldmann +- Cory Benfield (Lukasa) diff --git a/tools/jinja2/Jinja2-2.10.tar.gz.md5 b/tools/jinja2/Jinja2-2.10.tar.gz.md5 new file mode 100644 index 000000000000..9137ee129a0a --- /dev/null +++ b/tools/jinja2/Jinja2-2.10.tar.gz.md5 @@ -0,0 +1 @@ +61ef1117f945486472850819b8d1eb3d Jinja2-2.10.tar.gz diff --git a/tools/jinja2/Jinja2-2.10.tar.gz.sha512 b/tools/jinja2/Jinja2-2.10.tar.gz.sha512 new file mode 100644 index 000000000000..087d24c18eb8 --- /dev/null +++ b/tools/jinja2/Jinja2-2.10.tar.gz.sha512 @@ -0,0 +1 @@ +0ea7371be67ffcf19e46dfd06523a45a0806e678a407d54f5f2f3e573982f0959cf82ec5d07b203670309928a62ef71109701ab16547a9bba2ebcdc178cb67f2 Jinja2-2.10.tar.gz diff --git a/tools/jinja2/LICENSE b/tools/jinja2/LICENSE new file mode 100644 index 000000000000..31bf900e58e3 --- /dev/null +++ b/tools/jinja2/LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/jinja2/OWNERS b/tools/jinja2/OWNERS new file mode 100644 index 000000000000..ee2bec9ba3a3 --- /dev/null +++ b/tools/jinja2/OWNERS @@ -0,0 +1,6 @@ +timloh@chromium.org +haraken@chromium.org +nbarth@chromium.org + +# TEAM: platform-architecture-dev@chromium.org +# COMPONENT: Blink>Internals diff --git a/tools/jinja2/README.chromium b/tools/jinja2/README.chromium new file mode 100644 index 000000000000..5246c2f84b60 --- /dev/null +++ b/tools/jinja2/README.chromium @@ -0,0 +1,26 @@ +Name: Jinja2 Python Template Engine +Short Name: jinja2 +URL: http://jinja.pocoo.org/ +Version: 2.10 +License: BSD 3-Clause +License File: LICENSE +Security Critical: no + +Description: +Template engine for code generation in Blink. + +Source: https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz +MD5: 61ef1117f945486472850819b8d1eb3d +SHA-1: 34b69e5caab12ee37b9df69df9018776c008b7b8 + +Local Modifications: +This only includes the jinja2 directory from the tarball and the LICENSE and +AUTHORS files. Unit tests (testsuite directory) have been removed. +Additional chromium-specific files are: +* README.chromium (this file) +* OWNERS +* get_jinja2.sh (install script) +* jinja2.gni (generated by get_jinja2.sh) +* files of hashes (MD5 is also posted on website, SHA-512 computed locally). +Script checks hash then unpacks archive and installs desired files. +Retrieve or update by executing jinja2/get_jinja2.sh from third_party. diff --git a/tools/jinja2/__init__.py b/tools/jinja2/__init__.py new file mode 100644 index 000000000000..42aa763d571e --- /dev/null +++ b/tools/jinja2/__init__.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" + jinja2 + ~~~~~~ + + Jinja2 is a template engine written in pure Python. It provides a + Django inspired non-XML syntax but supports inline expressions and + an optional sandboxed environment. + + Nutshell + -------- + + Here a small example of a Jinja2 template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} + + {% endblock %} + + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +__docformat__ = 'restructuredtext en' +__version__ = '2.10' + +# high level interface +from jinja2.environment import Environment, Template + +# loaders +from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader + +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ + MemcachedBytecodeCache + +# undefined types +from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \ + make_logging_undefined + +# exceptions +from jinja2.exceptions import TemplateError, UndefinedError, \ + TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ + TemplateAssertionError, TemplateRuntimeError + +# decorators and public utilities +from jinja2.filters import environmentfilter, contextfilter, \ + evalcontextfilter +from jinja2.utils import Markup, escape, clear_caches, \ + environmentfunction, evalcontextfunction, contextfunction, \ + is_undefined, select_autoescape + +__all__ = [ + 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', + 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', + 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', + 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', + 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', + 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', + 'TemplateRuntimeError', + 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', + 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', + 'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined', + 'select_autoescape', +] + + +def _patch_async(): + from jinja2.utils import have_async_gen + if have_async_gen: + from jinja2.asyncsupport import patch_all + patch_all() + + +_patch_async() +del _patch_async diff --git a/tools/jinja2/_compat.py b/tools/jinja2/_compat.py new file mode 100644 index 000000000000..61d85301a4a9 --- /dev/null +++ b/tools/jinja2/_compat.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +""" + jinja2._compat + ~~~~~~~~~~~~~~ + + Some py2/py3 compatibility support based on a stripped down + version of six so we don't have to depend on a specific version + of it. + + :copyright: Copyright 2013 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, 'pypy_translation_info') +_identity = lambda x: x + + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + integer_types = (int,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + integer_types = (int, long) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO + + exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + + from itertools import imap, izip, ifilter + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + return cls + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode('utf-8') + return filename + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instantiation that replaces + # itself with the actual metaclass. + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote diff --git a/tools/jinja2/_identifier.py b/tools/jinja2/_identifier.py new file mode 100644 index 000000000000..2eac35d5c355 --- /dev/null +++ b/tools/jinja2/_identifier.py @@ -0,0 +1,2 @@ +# generated by scripts/generate_identifier_pattern.py +pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯' diff --git a/tools/jinja2/asyncfilters.py b/tools/jinja2/asyncfilters.py new file mode 100644 index 000000000000..5c1f46d7fa7d --- /dev/null +++ b/tools/jinja2/asyncfilters.py @@ -0,0 +1,146 @@ +from functools import wraps + +from jinja2.asyncsupport import auto_aiter +from jinja2 import filters + + +async def auto_to_seq(value): + seq = [] + if hasattr(value, '__aiter__'): + async for item in value: + seq.append(item) + else: + for item in value: + seq.append(item) + return seq + + +async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = filters.prepare_select_or_reject( + args, kwargs, modfunc, lookup_attr) + if seq: + async for item in auto_aiter(seq): + if func(item): + yield item + + +def dualfilter(normal_filter, async_filter): + wrap_evalctx = False + if getattr(normal_filter, 'environmentfilter', False): + is_async = lambda args: args[0].is_async + wrap_evalctx = False + else: + if not getattr(normal_filter, 'evalcontextfilter', False) and \ + not getattr(normal_filter, 'contextfilter', False): + wrap_evalctx = True + is_async = lambda args: args[0].environment.is_async + + @wraps(normal_filter) + def wrapper(*args, **kwargs): + b = is_async(args) + if wrap_evalctx: + args = args[1:] + if b: + return async_filter(*args, **kwargs) + return normal_filter(*args, **kwargs) + + if wrap_evalctx: + wrapper.evalcontextfilter = True + + wrapper.asyncfiltervariant = True + + return wrapper + + +def asyncfiltervariant(original): + def decorator(f): + return dualfilter(original, f) + return decorator + + +@asyncfiltervariant(filters.do_first) +async def do_first(environment, seq): + try: + return await auto_aiter(seq).__anext__() + except StopAsyncIteration: + return environment.undefined('No first item, sequence was empty.') + + +@asyncfiltervariant(filters.do_groupby) +async def do_groupby(environment, value, attribute): + expr = filters.make_attrgetter(environment, attribute) + return [filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby(sorted( + await auto_to_seq(value), key=expr), expr)] + + +@asyncfiltervariant(filters.do_join) +async def do_join(eval_ctx, value, d=u'', attribute=None): + return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) + + +@asyncfiltervariant(filters.do_list) +async def do_list(value): + return await auto_to_seq(value) + + +@asyncfiltervariant(filters.do_reject) +async def do_reject(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, False) + + +@asyncfiltervariant(filters.do_rejectattr) +async def do_rejectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, True) + + +@asyncfiltervariant(filters.do_select) +async def do_select(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, False) + + +@asyncfiltervariant(filters.do_selectattr) +async def do_selectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, True) + + +@asyncfiltervariant(filters.do_map) +async def do_map(*args, **kwargs): + seq, func = filters.prepare_map(args, kwargs) + if seq: + async for item in auto_aiter(seq): + yield func(item) + + +@asyncfiltervariant(filters.do_sum) +async def do_sum(environment, iterable, attribute=None, start=0): + rv = start + if attribute is not None: + func = filters.make_attrgetter(environment, attribute) + else: + func = lambda x: x + async for item in auto_aiter(iterable): + rv += func(item) + return rv + + +@asyncfiltervariant(filters.do_slice) +async def do_slice(value, slices, fill_with=None): + return filters.do_slice(await auto_to_seq(value), slices, fill_with) + + +ASYNC_FILTERS = { + 'first': do_first, + 'groupby': do_groupby, + 'join': do_join, + 'list': do_list, + # we intentionally do not support do_last because that would be + # ridiculous + 'reject': do_reject, + 'rejectattr': do_rejectattr, + 'map': do_map, + 'select': do_select, + 'selectattr': do_selectattr, + 'sum': do_sum, + 'slice': do_slice, +} diff --git a/tools/jinja2/asyncsupport.py b/tools/jinja2/asyncsupport.py new file mode 100644 index 000000000000..b1e7b5ce9a27 --- /dev/null +++ b/tools/jinja2/asyncsupport.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +""" + jinja2.asyncsupport + ~~~~~~~~~~~~~~~~~~~ + + Has all the code for async support which is implemented as a patch + for supported Python versions. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import sys +import asyncio +import inspect +from functools import update_wrapper + +from jinja2.utils import concat, internalcode, Markup +from jinja2.environment import TemplateModule +from jinja2.runtime import LoopContextBase, _last_iteration + + +async def concat_async(async_gen): + rv = [] + async def collect(): + async for event in async_gen: + rv.append(event) + await collect() + return concat(rv) + + +async def generate_async(self, *args, **kwargs): + vars = dict(*args, **kwargs) + try: + async for event in self.root_render_func(self.new_context(vars)): + yield event + except Exception: + exc_info = sys.exc_info() + else: + return + yield self.environment.handle_exception(exc_info, True) + + +def wrap_generate_func(original_generate): + def _convert_generator(self, loop, args, kwargs): + async_gen = self.generate_async(*args, **kwargs) + try: + while 1: + yield loop.run_until_complete(async_gen.__anext__()) + except StopAsyncIteration: + pass + def generate(self, *args, **kwargs): + if not self.environment.is_async: + return original_generate(self, *args, **kwargs) + return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + return update_wrapper(generate, original_generate) + + +async def render_async(self, *args, **kwargs): + if not self.environment.is_async: + raise RuntimeError('The environment was not created with async mode ' + 'enabled.') + + vars = dict(*args, **kwargs) + ctx = self.new_context(vars) + + try: + return await concat_async(self.root_render_func(ctx)) + except Exception: + exc_info = sys.exc_info() + return self.environment.handle_exception(exc_info, True) + + +def wrap_render_func(original_render): + def render(self, *args, **kwargs): + if not self.environment.is_async: + return original_render(self, *args, **kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.render_async(*args, **kwargs)) + return update_wrapper(render, original_render) + + +def wrap_block_reference_call(original_call): + @internalcode + async def async_call(self): + rv = await concat_async(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + @internalcode + def __call__(self): + if not self._context.environment.is_async: + return original_call(self) + return async_call(self) + + return update_wrapper(__call__, original_call) + + +def wrap_macro_invoke(original_invoke): + @internalcode + async def async_invoke(self, arguments, autoescape): + rv = await self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + @internalcode + def _invoke(self, arguments, autoescape): + if not self._environment.is_async: + return original_invoke(self, arguments, autoescape) + return async_invoke(self, arguments, autoescape) + return update_wrapper(_invoke, original_invoke) + + +@internalcode +async def get_default_module_async(self): + if self._module is not None: + return self._module + self._module = rv = await self.make_module_async() + return rv + + +def wrap_default_module(original_default_module): + @internalcode + def _get_default_module(self): + if self.environment.is_async: + raise RuntimeError('Template module attribute is unavailable ' + 'in async mode') + return original_default_module(self) + return _get_default_module + + +async def make_module_async(self, vars=None, shared=False, locals=None): + context = self.new_context(vars, shared, locals) + body_stream = [] + async for item in self.root_render_func(context): + body_stream.append(item) + return TemplateModule(self, context, body_stream) + + +def patch_template(): + from jinja2 import Template + Template.generate = wrap_generate_func(Template.generate) + Template.generate_async = update_wrapper( + generate_async, Template.generate_async) + Template.render_async = update_wrapper( + render_async, Template.render_async) + Template.render = wrap_render_func(Template.render) + Template._get_default_module = wrap_default_module( + Template._get_default_module) + Template._get_default_module_async = get_default_module_async + Template.make_module_async = update_wrapper( + make_module_async, Template.make_module_async) + + +def patch_runtime(): + from jinja2.runtime import BlockReference, Macro + BlockReference.__call__ = wrap_block_reference_call( + BlockReference.__call__) + Macro._invoke = wrap_macro_invoke(Macro._invoke) + + +def patch_filters(): + from jinja2.filters import FILTERS + from jinja2.asyncfilters import ASYNC_FILTERS + FILTERS.update(ASYNC_FILTERS) + + +def patch_all(): + patch_template() + patch_runtime() + patch_filters() + + +async def auto_await(value): + if inspect.isawaitable(value): + return await value + return value + + +async def auto_aiter(iterable): + if hasattr(iterable, '__aiter__'): + async for item in iterable: + yield item + return + for item in iterable: + yield item + + +class AsyncLoopContext(LoopContextBase): + + def __init__(self, async_iterator, undefined, after, length, recurse=None, + depth0=0): + LoopContextBase.__init__(self, undefined, recurse, depth0) + self._async_iterator = async_iterator + self._after = after + self._length = length + + @property + def length(self): + if self._length is None: + raise TypeError('Loop length for some iterators cannot be ' + 'lazily calculated in async mode') + return self._length + + def __aiter__(self): + return AsyncLoopContextIterator(self) + + +class AsyncLoopContextIterator(object): + __slots__ = ('context',) + + def __init__(self, context): + self.context = context + + def __aiter__(self): + return self + + async def __anext__(self): + ctx = self.context + ctx.index0 += 1 + if ctx._after is _last_iteration: + raise StopAsyncIteration() + ctx._before = ctx._current + ctx._current = ctx._after + try: + ctx._after = await ctx._async_iterator.__anext__() + except StopAsyncIteration: + ctx._after = _last_iteration + return ctx._current, ctx + + +async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): + # Length is more complicated and less efficient in async mode. The + # reason for this is that we cannot know if length will be used + # upfront but because length is a property we cannot lazily execute it + # later. This means that we need to buffer it up and measure :( + # + # We however only do this for actual iterators, not for async + # iterators as blocking here does not seem like the best idea in the + # world. + try: + length = len(iterable) + except (TypeError, AttributeError): + if not hasattr(iterable, '__aiter__'): + iterable = tuple(iterable) + length = len(iterable) + else: + length = None + async_iterator = auto_aiter(iterable) + try: + after = await async_iterator.__anext__() + except StopAsyncIteration: + after = _last_iteration + return AsyncLoopContext(async_iterator, undefined, after, length, recurse, + depth0) diff --git a/tools/jinja2/bccache.py b/tools/jinja2/bccache.py new file mode 100644 index 000000000000..080e527cabf3 --- /dev/null +++ b/tools/jinja2/bccache.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +""" + jinja2.bccache + ~~~~~~~~~~~~~~ + + This module implements the bytecode cache system Jinja is optionally + using. This is useful if you have very complex template situations and + the compiliation of all those templates slow down your application too + much. + + Situations where this is useful are often forking web applications that + are initialized on the first request. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD. +""" +from os import path, listdir +import os +import sys +import stat +import errno +import marshal +import tempfile +import fnmatch +from hashlib import sha1 +from jinja2.utils import open_if_exists +from jinja2._compat import BytesIO, pickle, PY2, text_type + + +# marshal works better on 3.x, one hack less required +if not PY2: + marshal_dump = marshal.dump + marshal_load = marshal.load +else: + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +bc_version = 3 + +# magic version used to only change with new jinja versions. With 2.6 +# we change this to also take Python version changes into account. The +# reason for this is that Python tends to segfault if fed earlier bytecode +# versions because someone thought it would be a good idea to reuse opcodes +# or make Python incompatible with earlier versions. +bc_magic = 'j2'.encode('ascii') + \ + pickle.dumps(bc_version, 2) + \ + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal_load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError('can\'t write empty bucket') + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + filename = '|' + filename + if isinstance(filename, text_type): + filename = filename.encode('utf-8') + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode('utf-8')).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self): + def _unsafe_dir(): + raise RuntimeError('Cannot determine safe temp directory. You ' + 'need to explicitly provide one.') + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == 'nt': + return tmpdir + if not hasattr(os, 'getuid'): + _unsafe_dir() + + dirname = '_jinja2-cache-%d' % os.getuid() + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if actual_dir_stat.st_uid != os.getuid() \ + or not stat.S_ISDIR(actual_dir_stat.st_mode) \ + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if actual_dir_stat.st_uid != os.getuid() \ + or not stat.S_ISDIR(actual_dir_stat.st_mode) \ + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), 'rb') + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), 'wb') + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `werkzeug `_.contrib.cache + - `python-memcached `_ + - `cmemcache `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, + ignore_memcache_errors=True): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/tools/jinja2/compiler.py b/tools/jinja2/compiler.py new file mode 100644 index 000000000000..d534a827391a --- /dev/null +++ b/tools/jinja2/compiler.py @@ -0,0 +1,1721 @@ +# -*- coding: utf-8 -*- +""" + jinja2.compiler + ~~~~~~~~~~~~~~~ + + Compiles nodes into python code. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from itertools import chain +from copy import deepcopy +from keyword import iskeyword as is_python_keyword +from functools import update_wrapper +from jinja2 import nodes +from jinja2.nodes import EvalContext +from jinja2.visitor import NodeVisitor +from jinja2.optimizer import Optimizer +from jinja2.exceptions import TemplateAssertionError +from jinja2.utils import Markup, concat, escape +from jinja2._compat import range_type, text_type, string_types, \ + iteritems, NativeStringIO, imap, izip +from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \ + VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED + + +operators = { + 'eq': '==', + 'ne': '!=', + 'gt': '>', + 'gteq': '>=', + 'lt': '<', + 'lteq': '<=', + 'in': 'in', + 'notin': 'not in' +} + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, 'iteritems'): + dict_item_iter = 'iteritems' +else: + dict_item_iter = 'items' + +code_features = ['division'] + +# does this python version support generator stops? (PEP 0479) +try: + exec('from __future__ import generator_stop') + code_features.append('generator_stop') +except SyntaxError: + pass + +# does this python version support yield from? +try: + exec('def f(): yield from x()') +except SyntaxError: + supports_yield_from = False +else: + supports_yield_from = True + + +def optimizeconst(f): + def new_func(self, node, frame, **kwargs): + # Only optimize if the frame is not volatile + if self.optimized and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: + return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) + return update_wrapper(new_func, f) + + +def generate(node, environment, name, filename, stream=None, + defer_init=False, optimized=True): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError('Can\'t compile non template nodes') + generator = environment.code_generator_class(environment, name, filename, + stream, defer_init, + optimized) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: + return True + if type(value) in (tuple, list, set, frozenset): + for item in value: + if not has_safe_repr(item): + return False + return True + elif type(value) is dict: + for key, value in iteritems(value): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef(object): + + def __init__(self, node): + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None, level=None): + self.eval_ctx = eval_ctx + self.symbols = Symbols(parent and parent.symbols or None, + level=level) + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated=False): + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == 'load' and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + + def __init__(self, environment, name, filename, stream=None, + defer_init=False, optimized=True): + if stream is None: + stream = NativeStringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimized = optimized + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack = [] + + # Tracks parameter definition blocks + self._param_def_block = [] + + # Tracks the current context. + self._context_reference_stack = ['context'] + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return 't_%d' % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline('%s = []' % frame.buffer) + + def return_buffer_contents(self, frame, force_unescaped=False): + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline('if context.eval_ctx.autoescape:') + self.indent() + self.writeline('return Markup(concat(%s))' % frame.buffer) + self.outdent() + self.writeline('else:') + self.indent() + self.writeline('return concat(%s)' % frame.buffer) + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline('return Markup(concat(%s))' % frame.buffer) + return + self.writeline('return concat(%s)' % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline('yield ', node) + else: + self.writeline('%s.append(' % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(')') + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline('pass') + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write('\n' * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, + self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(' ' * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occour. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(', ') + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(', ') + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write(', %s=%s' % (key, value)) + if node.dyn_args: + self.write(', *') + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(', **dict({') + else: + self.write(', **{') + for kwarg in node.kwargs: + self.write('%r: ' % kwarg.key) + self.visit(kwarg.value, frame) + self.write(', ') + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write('%r: %s, ' % (key, value)) + if node.dyn_kwargs is not None: + self.write('}, **') + self.visit(node.dyn_kwargs, frame) + self.write(')') + else: + self.write('}') + + elif node.dyn_kwargs is not None: + self.write(', **') + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in 'filters', 'tests': + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline('%s = environment.%s[%r]' % + (mapping[name], dependency, name)) + + def enter_frame(self, frame): + undefs = [] + for target, (action, param) in iteritems(frame.symbols.loads): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline('%s = %s(%r)' % + (target, self.get_resolve_func(), param)) + elif action == VAR_LOAD_ALIAS: + self.writeline('%s = %s' % (target, param)) + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError('unknown load instruction') + if undefs: + self.writeline('%s = missing' % ' = '.join(undefs)) + + def leave_frame(self, frame, with_python_scope=False): + if not with_python_scope: + undefs = [] + for target, _ in iteritems(frame.symbols.loads): + undefs.append(target) + if undefs: + self.writeline('%s = missing' % ' = '.join(undefs)) + + def func(self, name): + if self.environment.is_async: + return 'async def %s' % name + return 'def %s' % name + + def macro_body(self, node, frame): + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + for idx, arg in enumerate(node.args): + if arg.name == 'caller': + explicit_caller = idx + if arg.name in ('kwargs', 'varargs'): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ('caller', 'kwargs', 'varargs')) + + if 'caller' in undeclared: + # In older Jinja2 versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail('When defining macros or call blocks the ' + 'special "caller" argument must be omitted ' + 'or be given a default.', node.lineno) + else: + args.append(frame.symbols.declare_parameter('caller')) + macro_ref.accesses_caller = True + if 'kwargs' in undeclared and not 'kwargs' in skip_special_params: + args.append(frame.symbols.declare_parameter('kwargs')) + macro_ref.accesses_kwargs = True + if 'varargs' in undeclared and not 'varargs' in skip_special_params: + args.append(frame.symbols.declare_parameter('varargs')) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline('%s(%s):' % (self.func('macro'), ', '.join(args)), node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline('if %s is missing:' % ref) + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline('%s = undefined(%r, name=%r)' % ( + ref, + 'parameter %r was not provided' % arg.name, + arg.name)) + else: + self.writeline('%s = ' % ref) + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ', '.join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, 'name', None) + if len(macro_ref.node.args) == 1: + arg_tuple += ',' + self.write('Macro(environment, macro, %r, (%s), %r, %r, %r, ' + 'context.eval_ctx.autoescape)' % + (name, arg_tuple, macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, macro_ref.accesses_caller)) + + def position(self, node): + """Return a human readable position for the node.""" + rv = 'line %d' % node.lineno + if self.name is not None: + rv += ' in ' + repr(self.name) + return rv + + def dump_local_context(self, frame): + return '{%s}' % ', '.join( + '%r: %s' % (name, target) for name, target + in iteritems(frame.symbols.dump_stores())) + + def write_commons(self): + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline('resolve = context.resolve_or_missing') + self.writeline('undefined = environment.undefined') + self.writeline('if 0: yield None') + + def push_parameter_definitions(self, frame): + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self): + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target): + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target): + self._context_reference_stack.append(target) + + def pop_context_reference(self): + self._context_reference_stack.pop() + + def get_context_ref(self): + return self._context_reference_stack[-1] + + def get_resolve_func(self): + target = self._context_reference_stack[-1] + if target == 'context': + return 'resolve' + return '%s.resolve' % target + + def derive_context(self, frame): + return '%s.derived(%s)' % ( + self.get_context_ref(), + self.dump_local_context(frame), + ) + + def parameter_is_undeclared(self, target): + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self): + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame): + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if not frame.toplevel or not vars: + return + public_names = [x for x in vars if x[:1] != '_'] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + self.writeline('context.vars[%r] = %s' % (name, ref)) + else: + self.writeline('context.vars.update({') + for idx, name in enumerate(vars): + if idx: + self.write(', ') + ref = frame.symbols.ref(name) + self.write('%r: %s' % (name, ref)) + self.write('})') + if public_names: + if len(public_names) == 1: + self.writeline('context.exported_vars.add(%r)' % + public_names[0]) + else: + self.writeline('context.exported_vars.update((%s))' % + ', '.join(imap(repr, public_names))) + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, 'no root frame allowed' + eval_ctx = EvalContext(self.environment, self.name) + + from jinja2.runtime import __all__ as exported + self.writeline('from __future__ import %s' % ', '.join(code_features)) + self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + + if self.environment.is_async: + self.writeline('from jinja2.asyncsupport import auto_await, ' + 'auto_aiter, make_async_loop_context') + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ', environment=environment' or '' + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail('block %r defined twice' % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if '.' in imp: + module, obj = imp.rsplit('.', 1) + self.writeline('from %s import %s as %s' % + (module, obj, alias)) + else: + self.writeline('import %s as %s' % (imp, alias)) + + # add the load name + self.writeline('name = %r' % self.name) + + # generate the root render function. + self.writeline('%s(context, missing=missing%s):' % + (self.func('root'), envenv), extra=1) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if 'self' in find_undeclared(node.body, ('self',)): + ref = frame.symbols.declare_parameter('self') + self.writeline('%s = TemplateReference(context)' % ref) + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline('parent_template = None') + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline('if parent_template is not None:') + self.indent() + if supports_yield_from and not self.environment.is_async: + self.writeline('yield from parent_template.' + 'root_render_func(context)') + else: + self.writeline('%sfor event in parent_template.' + 'root_render_func(context):' % + (self.environment.is_async and 'async ' or '')) + self.indent() + self.writeline('yield event') + self.outdent() + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in iteritems(self.blocks): + self.writeline('%s(context, missing=missing%s):' % + (self.func('block_' + name), envenv), + block, 1) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + undeclared = find_undeclared(block.body, ('self', 'super')) + if 'self' in undeclared: + ref = block_frame.symbols.declare_parameter('self') + self.writeline('%s = TemplateReference(context)' % ref) + if 'super' in undeclared: + ref = block_frame.symbols.declare_parameter('super') + self.writeline('%s = context.super(%r, ' + 'block_%s)' % (ref, name, name)) + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) + for x in self.blocks), + extra=1) + + # add a function that returns the debug info + self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x + in self.debug_info)) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline('if parent_template is None:') + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if supports_yield_from and not self.environment.is_async and \ + frame.buffer is None: + self.writeline('yield from context.blocks[%r][0](%s)' % ( + node.name, context), node) + else: + loop = self.environment.is_async and 'async for' or 'for' + self.writeline('%s event in context.blocks[%r][0](%s):' % ( + loop, node.name, context), node) + self.indent() + self.simple_write('event', frame) + self.outdent() + + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail('cannot use extend from a non top-level scope', + node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'extended multiple times') + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline('parent_template = environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + self.writeline('for name, parent_block in parent_template.' + 'blocks.%s():' % dict_item_iter) + self.indent() + self.writeline('context.blocks.setdefault(name, []).' + 'append(parent_block)') + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.ignore_missing: + self.writeline('try:') + self.indent() + + func_name = 'get_or_select_template' + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, string_types): + func_name = 'get_template' + elif isinstance(node.template.value, (tuple, list)): + func_name = 'select_template' + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = 'select_template' + + self.writeline('template = environment.%s(' % func_name, node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + if node.ignore_missing: + self.outdent() + self.writeline('except TemplateNotFound:') + self.indent() + self.writeline('pass') + self.outdent() + self.writeline('else:') + self.indent() + + skip_event_yield = False + if node.with_context: + loop = self.environment.is_async and 'async for' or 'for' + self.writeline('%s event in template.root_render_func(' + 'template.new_context(context.get_all(), True, ' + '%s)):' % (loop, self.dump_local_context(frame))) + elif self.environment.is_async: + self.writeline('for event in (await ' + 'template._get_default_module_async())' + '._body_stream:') + else: + if supports_yield_from: + self.writeline('yield from template._get_default_module()' + '._body_stream') + skip_event_yield = True + else: + self.writeline('for event in template._get_default_module()' + '._body_stream:') + + if not skip_event_yield: + self.indent() + self.simple_write('event', frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline('%s = ' % frame.symbols.ref(node.target), node) + if frame.toplevel: + self.write('context.vars[%r] = ' % node.target) + if self.environment.is_async: + self.write('await ') + self.write('environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module%s(context.get_all(), True, %s)' + % (self.environment.is_async and '_async' or '', + self.dump_local_context(frame))) + elif self.environment.is_async: + self.write('_get_default_module_async()') + else: + self.write('_get_default_module()') + if frame.toplevel and not node.target.startswith('_'): + self.writeline('context.exported_vars.discard(%r)' % node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write('included_template = %senvironment.get_template(' + % (self.environment.is_async and 'await ' or '')) + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module%s(context.get_all(), True, %s)' + % (self.environment.is_async and '_async' or '', + self.dump_local_context(frame))) + elif self.environment.is_async: + self.write('_get_default_module_async()') + else: + self.write('_get_default_module()') + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline('%s = getattr(included_template, ' + '%r, missing)' % (frame.symbols.ref(alias), name)) + self.writeline('if %s is missing:' % frame.symbols.ref(alias)) + self.indent() + self.writeline('%s = undefined(%r %% ' + 'included_template.__name__, ' + 'name=%r)' % + (frame.symbols.ref(alias), + 'the template %%r (imported on %s) does ' + 'not export the requested name %s' % ( + self.position(node), + repr(name) + ), name)) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith('_'): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline('context.vars[%r] = %s' % + (name, frame.symbols.ref(name))) + else: + self.writeline('context.vars.update({%s})' % ', '.join( + '%r: %s' % (name, frame.symbols.ref(name)) for name in var_names + )) + if discarded_names: + if len(discarded_names) == 1: + self.writeline('context.exported_vars.discard(%r)' % + discarded_names[0]) + else: + self.writeline('context.exported_vars.difference_' + 'update((%s))' % ', '.join(imap(repr, discarded_names))) + + def visit_For(self, node, frame): + loop_frame = frame.inner() + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or 'loop' in \ + find_undeclared(node.iter_child_nodes( + only=('body',)), ('loop',)) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter('loop') + + loop_frame.symbols.analyze_node(node, for_branch='body') + if node.else_: + else_frame.symbols.analyze_node(node, for_branch='else') + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch='test') + self.writeline('%s(fiter):' % self.func(loop_filter_func), node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline(self.environment.is_async and 'async for ' or 'for ') + self.visit(node.target, loop_frame) + self.write(' in ') + self.write(self.environment.is_async and 'auto_aiter(fiter)' or 'fiter') + self.write(':') + self.indent() + self.writeline('if ', node.test) + self.visit(node.test, test_frame) + self.write(':') + self.indent() + self.writeline('yield ') + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline('%s(reciter, loop_render_func, depth=0):' % + self.func('loop'), node) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline('%s = missing' % loop_ref) + + for name in node.find_all(nodes.Name): + if name.ctx == 'store' and name.name == 'loop': + self.fail('Can\'t assign to special loop variable ' + 'in for-loop target', name.lineno) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline('%s = 1' % iteration_indicator) + + self.writeline(self.environment.is_async and 'async for ' or 'for ', node) + self.visit(node.target, loop_frame) + if extended_loop: + if self.environment.is_async: + self.write(', %s in await make_async_loop_context(' % loop_ref) + else: + self.write(', %s in LoopContext(' % loop_ref) + else: + self.write(' in ') + + if node.test: + self.write('%s(' % loop_filter_func) + if node.recursive: + self.write('reciter') + else: + if self.environment.is_async and not extended_loop: + self.write('auto_aiter(') + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(')') + if node.test: + self.write(')') + + if node.recursive: + self.write(', undefined, loop_render_func, depth):') + else: + self.write(extended_loop and ', undefined):' or ':') + + self.indent() + self.enter_frame(loop_frame) + + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline('%s = 0' % iteration_indicator) + self.outdent() + self.leave_frame(loop_frame, with_python_scope=node.recursive + and not node.else_) + + if node.else_: + self.writeline('if %s:' % iteration_indicator) + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + if self.environment.is_async: + self.write('await ') + self.write('loop(') + if self.environment.is_async: + self.write('auto_aiter(') + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(')') + self.write(', loop)') + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline('if ', node) + self.visit(node.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline('elif ', elif_) + self.visit(elif_.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline('else:') + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith('_'): + self.write('context.exported_vars.add(%r)' % node.name) + ref = frame.symbols.ref(node.name) + self.writeline('context.vars[%r] = ' % node.name) + self.write('%s = ' % frame.symbols.ref(node.name)) + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node, frame): + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline('caller = ') + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node, frame): + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for idx, (target, expr) in enumerate(izip(node.targets, node.values)): + self.newline() + self.visit(target, with_frame) + self.write(' = ') + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + def visit_Output(self, node, frame): + # if we have a known extends statement, we don't output anything + # if we are in a require_output_check section + if self.has_known_extends and frame.require_output_check: + return + + allow_constant_finalize = True + if self.environment.finalize: + func = self.environment.finalize + if getattr(func, 'contextfunction', False) or \ + getattr(func, 'evalcontextfunction', False): + allow_constant_finalize = False + elif getattr(func, 'environmentfunction', False): + finalize = lambda x: text_type( + self.environment.finalize(self.environment, x)) + else: + finalize = lambda x: text_type(self.environment.finalize(x)) + else: + finalize = text_type + + # if we are inside a frame that requires output checking, we do so + outdent_later = False + if frame.require_output_check: + self.writeline('if parent_template is None:') + self.indent() + outdent_later = True + + # try to evaluate as many chunks as possible into a static + # string at compile time. + body = [] + for child in node.nodes: + try: + if not allow_constant_finalize: + raise nodes.Impossible() + const = child.as_const(frame.eval_ctx) + except nodes.Impossible: + body.append(child) + continue + # the frame can't be volatile here, becaus otherwise the + # as_const() function would raise an Impossible exception + # at that point. + try: + if frame.eval_ctx.autoescape: + if hasattr(const, '__html__'): + const = const.__html__() + else: + const = escape(const) + const = finalize(const) + except Exception: + # if something goes wrong here we evaluate the node + # at runtime for easier debugging + body.append(child) + continue + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + # if we have less than 3 nodes or a buffer we yield or extend/append + if len(body) < 3 or frame.buffer is not None: + if frame.buffer is not None: + # for one item we append, for more we extend + if len(body) == 1: + self.writeline('%s.append(' % frame.buffer) + else: + self.writeline('%s.extend((' % frame.buffer) + self.indent() + for item in body: + if isinstance(item, list): + val = repr(concat(item)) + if frame.buffer is None: + self.writeline('yield ' + val) + else: + self.writeline(val + ',') + else: + if frame.buffer is None: + self.writeline('yield ', item) + else: + self.newline(item) + close = 1 + if frame.eval_ctx.volatile: + self.write('(escape if context.eval_ctx.autoescape' + ' else to_string)(') + elif frame.eval_ctx.autoescape: + self.write('escape(') + else: + self.write('to_string(') + if self.environment.finalize is not None: + self.write('environment.finalize(') + if getattr(self.environment.finalize, + "contextfunction", False): + self.write('context, ') + close += 1 + self.visit(item, frame) + self.write(')' * close) + if frame.buffer is not None: + self.write(',') + if frame.buffer is not None: + # close the open parentheses + self.outdent() + self.writeline(len(body) == 1 and ')' or '))') + + # otherwise we create a format string as this is faster in that case + else: + format = [] + arguments = [] + for item in body: + if isinstance(item, list): + format.append(concat(item).replace('%', '%%')) + else: + format.append('%s') + arguments.append(item) + self.writeline('yield ') + self.write(repr(concat(format)) + ' % (') + self.indent() + for argument in arguments: + self.newline(argument) + close = 0 + if frame.eval_ctx.volatile: + self.write('(escape if context.eval_ctx.autoescape else' + ' to_string)(') + close += 1 + elif frame.eval_ctx.autoescape: + self.write('escape(') + close += 1 + if self.environment.finalize is not None: + self.write('environment.finalize(') + if getattr(self.environment.finalize, + 'contextfunction', False): + self.write('context, ') + elif getattr(self.environment.finalize, + 'evalcontextfunction', False): + self.write('context.eval_ctx, ') + elif getattr(self.environment.finalize, + 'environmentfunction', False): + self.write('environment, ') + close += 1 + self.visit(argument, frame) + self.write(')' * close + ', ') + self.outdent() + self.writeline(')') + + if outdent_later: + self.outdent() + + def visit_Assign(self, node, frame): + self.push_assign_tracking() + self.newline(node) + self.visit(node.target, frame) + self.write(' = ') + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node, frame): + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(' = (Markup if context.eval_ctx.autoescape ' + 'else identity)(') + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write('concat(%s)' % block_frame.buffer) + self.write(')') + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == 'store' and frame.toplevel: + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == 'load': + load = frame.symbols.find_load(ref) + if not (load is not None and load[0] == VAR_LOAD_PARAMETER and \ + not self.parameter_is_undeclared(ref)): + self.write('(undefined(name=%r) if %s is missing else %s)' % + (node.name, ref, ref)) + return + + self.write(ref) + + def visit_NSRef(self, node, frame): + # NSRefs can only be used to store values; since they use the normal + # `foo.bar` notation they will be parsed as a normal attribute access + # when used anywhere but in a `set` context + ref = frame.symbols.ref(node.name) + self.writeline('if not isinstance(%s, Namespace):' % ref) + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'cannot assign attribute on non-namespace object') + self.outdent() + self.writeline('%s[%r]' % (ref, node.attr)) + + def visit_Const(self, node, frame): + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write('(Markup if context.eval_ctx.autoescape else identity)(%r)' + % node.data) + + def visit_Tuple(self, node, frame): + self.write('(') + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(idx == 0 and ',)' or ')') + + def visit_List(self, node, frame): + self.write('[') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(']') + + def visit_Dict(self, node, frame): + self.write('{') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item.key, frame) + self.write(': ') + self.visit(item.value, frame) + self.write('}') + + def binop(operator, interceptable=True): + @optimizeconst + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_binops: + self.write('environment.call_binop(context, %r, ' % operator) + self.visit(node.left, frame) + self.write(', ') + self.visit(node.right, frame) + else: + self.write('(') + self.visit(node.left, frame) + self.write(' %s ' % operator) + self.visit(node.right, frame) + self.write(')') + return visitor + + def uaop(operator, interceptable=True): + @optimizeconst + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_unops: + self.write('environment.call_unop(context, %r, ' % operator) + self.visit(node.node, frame) + else: + self.write('(' + operator) + self.visit(node.node, frame) + self.write(')') + return visitor + + visit_Add = binop('+') + visit_Sub = binop('-') + visit_Mul = binop('*') + visit_Div = binop('/') + visit_FloorDiv = binop('//') + visit_Pow = binop('**') + visit_Mod = binop('%') + visit_And = binop('and', interceptable=False) + visit_Or = binop('or', interceptable=False) + visit_Pos = uaop('+') + visit_Neg = uaop('-') + visit_Not = uaop('not ', interceptable=False) + del binop, uaop + + @optimizeconst + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = '(context.eval_ctx.volatile and' \ + ' markup_join or unicode_join)' + elif frame.eval_ctx.autoescape: + func_name = 'markup_join' + else: + func_name = 'unicode_join' + self.write('%s((' % func_name) + for arg in node.nodes: + self.visit(arg, frame) + self.write(', ') + self.write('))') + + @optimizeconst + def visit_Compare(self, node, frame): + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + + def visit_Operand(self, node, frame): + self.write(' %s ' % operators[node.op]) + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node, frame): + self.write('environment.getattr(') + self.visit(node.node, frame) + self.write(', %r)' % node.attr) + + @optimizeconst + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write('[') + self.visit(node.arg, frame) + self.write(']') + else: + self.write('environment.getitem(') + self.visit(node.node, frame) + self.write(', ') + self.visit(node.arg, frame) + self.write(')') + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(':') + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(':') + self.visit(node.step, frame) + + @optimizeconst + def visit_Filter(self, node, frame): + if self.environment.is_async: + self.write('await auto_await(') + self.write(self.filters[node.name] + '(') + func = self.environment.filters.get(node.name) + if func is None: + self.fail('no filter named %r' % node.name, node.lineno) + if getattr(func, 'contextfilter', False): + self.write('context, ') + elif getattr(func, 'evalcontextfilter', False): + self.write('context.eval_ctx, ') + elif getattr(func, 'environmentfilter', False): + self.write('environment, ') + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write('(context.eval_ctx.autoescape and' + ' Markup(concat(%s)) or concat(%s))' % + (frame.buffer, frame.buffer)) + elif frame.eval_ctx.autoescape: + self.write('Markup(concat(%s))' % frame.buffer) + else: + self.write('concat(%s)' % frame.buffer) + self.signature(node, frame) + self.write(')') + if self.environment.is_async: + self.write(')') + + @optimizeconst + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + '(') + if node.name not in self.environment.tests: + self.fail('no test named %r' % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(')') + + @optimizeconst + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write('undefined(%r)' % ('the inline if-' + 'expression on %s evaluated to false and ' + 'no else section was defined.' % self.position(node))) + + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') + + @optimizeconst + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.is_async: + self.write('await auto_await(') + if self.environment.sandboxed: + self.write('environment.call(context, ') + else: + self.write('context.call(') + self.visit(node.node, frame) + extra_kwargs = forward_caller and {'caller': 'caller'} or None + self.signature(node, frame, extra_kwargs) + self.write(')') + if self.environment.is_async: + self.write(')') + + def visit_Keyword(self, node, frame): + self.write(node.key + '=') + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write('Markup(') + self.visit(node.expr, frame) + self.write(')') + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write('(context.eval_ctx.autoescape and Markup or identity)(') + self.visit(node.expr, frame) + self.write(')') + + def visit_EnvironmentAttribute(self, node, frame): + self.write('environment.' + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write('context') + + def visit_Continue(self, node, frame): + self.writeline('continue', node) + + def visit_Break(self, node, frame): + self.writeline('break', node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node, frame): + ctx = self.temporary_identifier() + self.writeline('%s = %s' % (ctx, self.derive_context(frame))) + self.writeline('%s.vars = ' % ctx) + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline('context.eval_ctx.%s = ' % keyword.key) + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline('%s = context.eval_ctx.save()' % old_ctx_name) + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name) diff --git a/tools/jinja2/constants.py b/tools/jinja2/constants.py new file mode 100644 index 000000000000..11efd1ed1583 --- /dev/null +++ b/tools/jinja2/constants.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" + jinja.constants + ~~~~~~~~~~~~~~~ + + Various constants. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" + + +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = u'''\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate''' diff --git a/tools/jinja2/debug.py b/tools/jinja2/debug.py new file mode 100644 index 000000000000..b61139f0cde1 --- /dev/null +++ b/tools/jinja2/debug.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +""" + jinja2.debug + ~~~~~~~~~~~~ + + Implements the debug interface for Jinja. This module does some pretty + ugly stuff with the Python traceback system in order to achieve tracebacks + with correct line numbers, locals and contents. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import sys +import traceback +from types import TracebackType, CodeType +from jinja2.utils import missing, internal_code +from jinja2.exceptions import TemplateSyntaxError +from jinja2._compat import iteritems, reraise, PY2 + +# on pypy we can take advantage of transparent proxies +try: + from __pypy__ import tproxy +except ImportError: + tproxy = None + + +# how does the raise helper look like? +try: + exec("raise TypeError, 'foo'") +except SyntaxError: + raise_helper = 'raise __jinja_exception__[1]' +except TypeError: + raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]' + + +class TracebackFrameProxy(object): + """Proxies a traceback frame.""" + + def __init__(self, tb): + self.tb = tb + self._tb_next = None + + @property + def tb_next(self): + return self._tb_next + + def set_next(self, next): + if tb_set_next is not None: + try: + tb_set_next(self.tb, next and next.tb or None) + except Exception: + # this function can fail due to all the hackery it does + # on various python implementations. We just catch errors + # down and ignore them if necessary. + pass + self._tb_next = next + + @property + def is_jinja_frame(self): + return '__jinja_template__' in self.tb.tb_frame.f_globals + + def __getattr__(self, name): + return getattr(self.tb, name) + + +def make_frame_proxy(frame): + proxy = TracebackFrameProxy(frame) + if tproxy is None: + return proxy + def operation_handler(operation, *args, **kwargs): + if operation in ('__getattribute__', '__getattr__'): + return getattr(proxy, args[0]) + elif operation == '__setattr__': + proxy.__setattr__(*args, **kwargs) + else: + return getattr(proxy, operation)(*args, **kwargs) + return tproxy(TracebackType, operation_handler) + + +class ProcessedTraceback(object): + """Holds a Jinja preprocessed traceback for printing or reraising.""" + + def __init__(self, exc_type, exc_value, frames): + assert frames, 'no frames for this traceback?' + self.exc_type = exc_type + self.exc_value = exc_value + self.frames = frames + + # newly concatenate the frames (which are proxies) + prev_tb = None + for tb in self.frames: + if prev_tb is not None: + prev_tb.set_next(tb) + prev_tb = tb + prev_tb.set_next(None) + + def render_as_text(self, limit=None): + """Return a string with the traceback.""" + lines = traceback.format_exception(self.exc_type, self.exc_value, + self.frames[0], limit=limit) + return ''.join(lines).rstrip() + + def render_as_html(self, full=False): + """Return a unicode string with the traceback as rendered HTML.""" + from jinja2.debugrenderer import render_traceback + return u'%s\n\n' % ( + render_traceback(self, full=full), + self.render_as_text().decode('utf-8', 'replace') + ) + + @property + def is_template_syntax_error(self): + """`True` if this is a template syntax error.""" + return isinstance(self.exc_value, TemplateSyntaxError) + + @property + def exc_info(self): + """Exception info tuple with a proxy around the frame objects.""" + return self.exc_type, self.exc_value, self.frames[0] + + @property + def standard_exc_info(self): + """Standard python exc_info for re-raising""" + tb = self.frames[0] + # the frame will be an actual traceback (or transparent proxy) if + # we are on pypy or a python implementation with support for tproxy + if type(tb) is not TracebackType: + tb = tb.tb + return self.exc_type, self.exc_value, tb + + +def make_traceback(exc_info, source_hint=None): + """Creates a processed traceback object from the exc_info.""" + exc_type, exc_value, tb = exc_info + if isinstance(exc_value, TemplateSyntaxError): + exc_info = translate_syntax_error(exc_value, source_hint) + initial_skip = 0 + else: + initial_skip = 1 + return translate_exception(exc_info, initial_skip) + + +def translate_syntax_error(error, source=None): + """Rewrites a syntax error to please traceback systems.""" + error.source = source + error.translated = True + exc_info = (error.__class__, error, None) + filename = error.filename + if filename is None: + filename = '' + return fake_exc_info(exc_info, filename, error.lineno) + + +def translate_exception(exc_info, initial_skip=0): + """If passed an exc_info it will automatically rewrite the exceptions + all the way down to the correct line numbers and frames. + """ + tb = exc_info[2] + frames = [] + + # skip some internal frames if wanted + for x in range(initial_skip): + if tb is not None: + tb = tb.tb_next + initial_tb = tb + + while tb is not None: + # skip frames decorated with @internalcode. These are internal + # calls we can't avoid and that are useless in template debugging + # output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + # save a reference to the next frame if we override the current + # one with a faked one. + next = tb.tb_next + + # fake template exceptions + template = tb.tb_frame.f_globals.get('__jinja_template__') + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, + lineno)[2] + + frames.append(make_frame_proxy(tb)) + tb = next + + # if we don't have any exceptions in the frames left, we have to + # reraise it unchanged. + # XXX: can we backup here? when could this happen? + if not frames: + reraise(exc_info[0], exc_info[1], exc_info[2]) + + return ProcessedTraceback(exc_info[0], exc_info[1], frames) + + +def get_jinja_locals(real_locals): + ctx = real_locals.get('context') + if ctx: + locals = ctx.get_all().copy() + else: + locals = {} + + local_overrides = {} + + for name, value in iteritems(real_locals): + if not name.startswith('l_') or value is missing: + continue + try: + _, depth, name = name.split('_', 2) + depth = int(depth) + except ValueError: + continue + cur_depth = local_overrides.get(name, (-1,))[0] + if cur_depth < depth: + local_overrides[name] = (depth, value) + + for name, (_, value) in iteritems(local_overrides): + if value is missing: + locals.pop(name, None) + else: + locals[name] = value + + return locals + + +def fake_exc_info(exc_info, filename, lineno): + """Helper for `translate_exception`.""" + exc_type, exc_value, tb = exc_info + + # figure the real context out + if tb is not None: + locals = get_jinja_locals(tb.tb_frame.f_locals) + + # if there is a local called __jinja_exception__, we get + # rid of it to not break the debug functionality. + locals.pop('__jinja_exception__', None) + else: + locals = {} + + # assamble fake globals we need + globals = { + '__name__': filename, + '__file__': filename, + '__jinja_exception__': exc_info[:2], + + # we don't want to keep the reference to the template around + # to not cause circular dependencies, but we mark it as Jinja + # frame for the ProcessedTraceback + '__jinja_template__': None + } + + # and fake the exception + code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') + + # if it's possible, change the name of the code. This won't work + # on some python environments such as google appengine + try: + if tb is None: + location = 'template' + else: + function = tb.tb_frame.f_code.co_name + if function == 'root': + location = 'top-level template code' + elif function.startswith('block_'): + location = 'block "%s"' % function[6:] + else: + location = 'template' + + if PY2: + code = CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + else: + code = CodeType(0, code.co_kwonlyargcount, + code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + except Exception as e: + pass + + # execute the code and catch the new traceback + try: + exec(code, globals, locals) + except: + exc_info = sys.exc_info() + new_tb = exc_info[2].tb_next + + # return without this frame + return exc_info[:2] + (new_tb,) + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. Do not attempt to use this on non cpython + interpreters + """ + import ctypes + from types import TracebackType + + if PY2: + # figure out size of _Py_ssize_t for Python 2: + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + else: + # platform ssize_t on Python 3 + _Py_ssize_t = ctypes.c_ssize_t + + # regular python + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if hasattr(sys, 'getobjects'): + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and + (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +# try to get a tb_set_next implementation if we don't have transparent +# proxies. +tb_set_next = None +if tproxy is None: + try: + tb_set_next = _init_ugly_crap() + except: + pass + del _init_ugly_crap diff --git a/tools/jinja2/defaults.py b/tools/jinja2/defaults.py new file mode 100644 index 000000000000..7c93dec0aeb2 --- /dev/null +++ b/tools/jinja2/defaults.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" + jinja2.defaults + ~~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from jinja2._compat import range_type +from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace + + +# defaults for the parser / lexer +BLOCK_START_STRING = '{%' +BLOCK_END_STRING = '%}' +VARIABLE_START_STRING = '{{' +VARIABLE_END_STRING = '}}' +COMMENT_START_STRING = '{#' +COMMENT_END_STRING = '#}' +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False + + +# default filters, tests and namespace +from jinja2.filters import FILTERS as DEFAULT_FILTERS +from jinja2.tests import TESTS as DEFAULT_TESTS +DEFAULT_NAMESPACE = { + 'range': range_type, + 'dict': dict, + 'lipsum': generate_lorem_ipsum, + 'cycler': Cycler, + 'joiner': Joiner, + 'namespace': Namespace +} + + +# default policies +DEFAULT_POLICIES = { + 'compiler.ascii_str': True, + 'urlize.rel': 'noopener', + 'urlize.target': None, + 'truncate.leeway': 5, + 'json.dumps_function': None, + 'json.dumps_kwargs': {'sort_keys': True}, + 'ext.i18n.trimmed': False, +} + + +# export all constants +__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/tools/jinja2/environment.py b/tools/jinja2/environment.py new file mode 100644 index 000000000000..549d9afab456 --- /dev/null +++ b/tools/jinja2/environment.py @@ -0,0 +1,1276 @@ +# -*- coding: utf-8 -*- +""" + jinja2.environment + ~~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import sys +import weakref +from functools import reduce, partial +from jinja2 import nodes +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ + DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS +from jinja2.lexer import get_lexer, TokenStream +from jinja2.parser import Parser +from jinja2.nodes import EvalContext +from jinja2.compiler import generate, CodeGenerator +from jinja2.runtime import Undefined, new_context, Context +from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ + TemplatesNotFound, TemplateRuntimeError +from jinja2.utils import import_string, LRUCache, Markup, missing, \ + concat, consume, internalcode, have_async_gen +from jinja2._compat import imap, ifilter, string_types, iteritems, \ + text_type, reraise, implements_iterator, implements_to_string, \ + encode_filename, PY2, PYPY + + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + +# the function to create jinja traceback objects. This is dynamically +# imported on the first exception in the exception handler. +_make_traceback = None + + +def get_spontaneous_environment(*args): + """Return a new spontaneous environment. A spontaneous environment is an + unnamed and unaccessible (in theory) environment that is used for + templates generated from a string and not from the file system. + """ + try: + env = _spontaneous_environments.get(args) + except TypeError: + return Environment(*args) + if env is not None: + return env + _spontaneous_environments[args] = env = Environment(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(string, name): + msg = string % name + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e) + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass(environment.undefined, Undefined), 'undefined must ' \ + 'be a subclass of undefined because filters depend on it.' + assert environment.block_start_string != \ + environment.variable_start_string != \ + environment.comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' + assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ + 'newline_sequence set to unknown line ending string.' + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which allows + you to take advantage of newer Python features. This requires + Python 3.6 or later. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: these are currently EXPERIMENTAL undocumented features. + exception_handler = None + exception_formatter = None + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__(self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay(self, block_start_string=missing, block_end_string=missing, + variable_start_string=missing, variable_end_string=missing, + comment_start_string=missing, comment_end_string=missing, + line_statement_prefix=missing, line_comment_prefix=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, + undefined=missing, finalize=missing, autoescape=missing, + loader=missing, cache_size=missing, auto_reload=missing, + bytecode_cache=missing): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args['self'], args['cache_size'], args['extensions'] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), + key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + Note that on Python 3 this might return a coroutine in case the + filter is running from an environment in async mode and the filter + supports async execution. It's your responsibility to await this + if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable('no filter named %r', name) + args = [value] + list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable('no test named %r', name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja2 extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce(lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), text_type(source)) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate(source, self, name, filename, defer_init=defer_init, + optimized=self.optimized) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, 'exec') + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, + defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, + defer_init=defer_init) + if raw: + return source + if filename is None: + filename = '