diff --git a/LICENSE b/LICENSE index 5dffdb791b08..7fc997b4f04d 100644 --- a/LICENSE +++ b/LICENSE @@ -1035,6 +1035,109 @@ The externally maintained libraries used by Node.js are: OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ +- inspector_protocol, located at tools/inspector_protocol, is licensed as follows: + """ + // Copyright 2016 The Chromium Authors. All rights reserved. + // + // Redistribution and use in source and binary forms, with or without + // modification, are permitted provided that the following conditions are + // met: + // + // * Redistributions of source code must retain the above copyright + // notice, this list of conditions and the following disclaimer. + // * Redistributions in binary form must reproduce the above + // copyright notice, this list of conditions and the following disclaimer + // in the documentation and/or other materials provided with the + // distribution. + // * Neither the name of Google Inc. nor the names of its + // contributors may be used to endorse or promote products derived from + // this software without specific prior written permission. + // + // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- jinja2, located at tools/jinja2, is licensed as follows: + """ + Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- markupsafe, located at tools/markupsafe, is licensed as follows: + """ + Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS + for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms of the software as well + as documentation, with or without modification, are permitted provided + that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + """ + - cpplint.py, located at tools/cpplint.py, is licensed as follows: """ Copyright (c) 2009 Google Inc. All rights reserved. diff --git a/Makefile b/Makefile index fc4a094656a4..525e28e5efbb 100644 --- a/Makefile +++ b/Makefile @@ -320,10 +320,16 @@ ifeq ($(OSTYPE),aix) DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp endif +node_use_openssl = $(shell $(call available-node,"-p" \ + "process.versions.openssl != undefined")) test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules +ifeq ($(node_use_openssl),true) $(RM) -r test/addons/??_*/ [ -x $(NODE) ] && $(NODE) $< || node $< touch $@ +else + @echo "Skipping .docbuildstamp (no crypto)" +endif ADDONS_BINDING_GYPS := \ $(filter-out test/addons/??_*/binding.gyp, \ @@ -333,19 +339,26 @@ ADDONS_BINDING_SOURCES := \ $(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \ $(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h)) +ADDONS_PREREQS := config.gypi \ + deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ + deps/uv/include/*.h deps/v8/include/*.h \ + src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h + +define run_build_addons +env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ + npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ + "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \ + $1 +touch $2 +endef + # Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale. # Depends on node-gyp package.json so that build-addons is (re)executed when # node-gyp is updated as part of an npm update. -test/addons/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ +test/addons/.buildstamp: $(ADDONS_PREREQS) \ $(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ test/addons/.docbuildstamp - env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ - npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ - "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" "$$PWD/test/addons" - touch $@ + @$(call run_build_addons,"$$PWD/test/addons",$@) .PHONY: build-addons # .buildstamp needs $(NODE_EXE) but cannot depend on it @@ -366,17 +379,10 @@ ADDONS_NAPI_BINDING_SOURCES := \ $(filter-out test/addons-napi/??_*/*.h, $(wildcard test/addons-napi/*/*.h)) # Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale. -test/addons-napi/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \ +test/addons-napi/.buildstamp: $(ADDONS_PREREQS) \ $(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ src/node_api.h src/node_api_types.h - env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ - npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \ - "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \ - "$$PWD/test/addons-napi" - touch $@ + @$(call run_build_addons,"$$PWD/test/addons-napi",$@) .PHONY: build-addons-napi # .buildstamp needs $(NODE_EXE) but cannot depend on it @@ -1060,17 +1066,18 @@ lint-md-build: tools/remark-cli/node_modules \ tools/doc/node_modules \ tools/remark-preset-lint-node/node_modules -.PHONY: tools/doc/node_modules -tools/doc/node_modules: - @cd tools/doc && $(call available-node,$(run-npm-install)) +tools/doc/node_modules: tools/doc/package.json +ifeq ($(node_use_openssl),true) + cd tools/doc && $(call available-node,$(run-npm-install)) +else + @echo "Skipping tools/doc/node_modules (no crypto)" +endif .PHONY: lint-md ifneq ("","$(wildcard tools/remark-cli/node_modules/)") LINT_MD_DOC_FILES = $(shell ls doc/*.md doc/**/*.md) run-lint-doc-md = tools/remark-cli/cli.js -q -f $(LINT_MD_DOC_FILES) -node_use_openssl = $(shell $(call available-node,"-p" \ - "process.versions.openssl != undefined")) # Lint all changed markdown files under doc/ tools/.docmdlintstamp: $(LINT_MD_DOC_FILES) ifeq ($(node_use_openssl),true) diff --git a/README.md b/README.md index 925c9e39b8c1..f7ef7c66b191 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,8 @@ For more information about the governance of the Node.js project, see **Jeremiah Senkpiel** <fishrock123@rocketmail.com> * [gabrielschulhof](https://github.com/gabrielschulhof) - **Gabriel Schulhof** <gabriel.schulhof@intel.com> +* [gdams](https://github.com/gdams) - +**George Adams** <george.adams@uk.ibm.com> (he/him) * [geek](https://github.com/geek) - **Wyatt Preul** <wpreul@gmail.com> * [gibfahn](https://github.com/gibfahn) - diff --git a/benchmark/fs/bench-mkdirp.js b/benchmark/fs/bench-mkdirp.js new file mode 100644 index 000000000000..96a792c35a48 --- /dev/null +++ b/benchmark/fs/bench-mkdirp.js @@ -0,0 +1,23 @@ +'use strict'; + +const common = require('../common'); +const fs = require('fs'); +const tmpdir = require('../../test/common/tmpdir'); +tmpdir.refresh(); +let dirc = 0; + +const bench = common.createBenchmark(main, { + n: [1e4], +}); + +function main({ n }) { + bench.start(); + (function r(cntr) { + if (cntr-- <= 0) + return bench.end(n); + const pathname = `${tmpdir.path}/${++dirc}/${++dirc}/${++dirc}/${++dirc}`; + fs.mkdir(pathname, { createParents: true }, (err) => { + r(cntr); + }); + }(n)); +} diff --git a/benchmark/misc/trace.js b/benchmark/misc/trace.js index 7ac80ca4cf85..fb51c91e9e3e 100644 --- a/benchmark/misc/trace.js +++ b/benchmark/misc/trace.js @@ -4,7 +4,7 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { n: [100000], - method: ['trace', 'emit', 'isTraceCategoryEnabled', 'categoryGroupEnabled'] + method: ['trace', 'isTraceCategoryEnabled'] }, { flags: ['--expose-internals', '--trace-event-categories', 'foo'] }); @@ -13,14 +13,6 @@ const { TRACE_EVENT_PHASE_NESTABLE_ASYNC_BEGIN: kBeforeEvent } = process.binding('constants').trace; -function doEmit(n, emit) { - bench.start(); - for (var i = 0; i < n; i++) { - emit(kBeforeEvent, 'foo', 'test', 0, 'arg1', 1); - } - bench.end(n); -} - function doTrace(n, trace) { bench.start(); for (var i = 0; i < n; i++) { @@ -38,23 +30,12 @@ function doIsTraceCategoryEnabled(n, isTraceCategoryEnabled) { bench.end(n); } -function doCategoryGroupEnabled(n, categoryGroupEnabled) { - bench.start(); - for (var i = 0; i < n; i++) { - categoryGroupEnabled('foo'); - categoryGroupEnabled('bar'); - } - bench.end(n); -} - function main({ n, method }) { const { internalBinding } = require('internal/test/binding'); const { trace, - isTraceCategoryEnabled, - emit, - categoryGroupEnabled + isTraceCategoryEnabled } = internalBinding('trace_events'); switch (method) { @@ -62,15 +43,9 @@ function main({ n, method }) { case 'trace': doTrace(n, trace); break; - case 'emit': - doEmit(n, emit); - break; case 'isTraceCategoryEnabled': doIsTraceCategoryEnabled(n, isTraceCategoryEnabled); break; - case 'categoryGroupEnabled': - doCategoryGroupEnabled(n, categoryGroupEnabled); - break; default: throw new Error(`Unexpected method "${method}"`); } diff --git a/doc/api/child_process.md b/doc/api/child_process.md index a2e6bb5365cf..ee75cbc5f348 100644 --- a/doc/api/child_process.md +++ b/doc/api/child_process.md @@ -536,11 +536,12 @@ process will be made the leader of a new process group and session. Note that child processes may continue running after the parent exits regardless of whether they are detached or not. See setsid(2) for more information. -By default, the parent will wait for the detached child to exit. To prevent -the parent from waiting for a given `subprocess`, use the `subprocess.unref()` -method. Doing so will cause the parent's event loop to not include the child in -its reference count, allowing the parent to exit independently of the child, -unless there is an established IPC channel between the child and parent. +By default, the parent will wait for the detached child to exit. To prevent the +parent from waiting for a given `subprocess` to exit, use the +`subprocess.unref()` method. Doing so will cause the parent's event loop to not +include the child in its reference count, allowing the parent to exit +independently of the child, unless there is an established IPC channel between +the child and the parent. When using the `detached` option to start a long-running process, the process will not stay running in the background after the parent exits unless it is @@ -1094,6 +1095,27 @@ console.log(`Spawned child pid: ${grep.pid}`); grep.stdin.end(); ``` +### subprocess.ref() + + +Calling `subprocess.ref()` after making a call to `subprocess.unref()` will +restore the removed reference count for the child process, forcing the parent +to wait for the child to exit before exiting itself. + +```js +const { spawn } = require('child_process'); + +const subprocess = spawn(process.argv[0], ['child_program.js'], { + detached: true, + stdio: 'ignore' +}); + +subprocess.unref(); +subprocess.ref(); +``` + ### subprocess.send(message[, sendHandle[, options]][, callback]) + +By default, the parent will wait for the detached child to exit. To prevent the +parent from waiting for a given `subprocess` to exit, use the +`subprocess.unref()` method. Doing so will cause the parent's event loop to not +include the child in its reference count, allowing the parent to exit +independently of the child, unless there is an established IPC channel between +the child and the parent. + +```js +const { spawn } = require('child_process'); + +const subprocess = spawn(process.argv[0], ['child_program.js'], { + detached: true, + stdio: 'ignore' +}); + +subprocess.unref(); +``` + ## `maxBuffer` and Unicode The `maxBuffer` option specifies the largest number of bytes allowed on `stdout` diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md index 6262fe4cf54e..56935574e618 100644 --- a/doc/api/deprecations.md +++ b/doc/api/deprecations.md @@ -1021,6 +1021,16 @@ accessed outside of Node.js core: `Socket.prototype._handle`, `Socket.prototype._healthCheck()`, `Socket.prototype._stopReceiving()`, and `dgram._createSocketHandle()`. + +### DEP0113: Cipher.setAuthTag(), Decipher.getAuthTag() + +Type: Runtime + +With the current crypto API, having `Cipher.setAuthTag()` and +`Decipher.getAuthTag()` is not helpful and both functions will throw an error +when called. They have never been documented and will be removed in a future +release. + [`--pending-deprecation`]: cli.html#cli_pending_deprecation [`Buffer.allocUnsafeSlow(size)`]: buffer.html#buffer_class_method_buffer_allocunsafeslow_size [`Buffer.from(array)`]: buffer.html#buffer_class_method_buffer_from_array diff --git a/doc/api/fs.md b/doc/api/fs.md index d0348d45c93f..1f71a537e3eb 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -2047,7 +2047,7 @@ changes: Synchronous lstat(2). -## fs.mkdir(path[, mode], callback) +## fs.mkdir(path[, options], callback) * `path` {string|Buffer|URL} -* `mode` {integer} Not supported on Windows. **Default:** `0o777`. +* `options` {Object|integer} + * `recursive` {boolean} **Default:** `false` + * `mode` {integer} Not supported on Windows. **Default:** `0o777`. * `callback` {Function} * `err` {Error} Asynchronously creates a directory. No arguments other than a possible exception are given to the completion callback. +The optional `options` argument can be an integer specifying mode (permission +and sticky bits), or an object with a `mode` property and a `recursive` +property indicating whether parent folders should be created. + +```js +// Creates /tmp/a/apple, regardless of whether `/tmp` and /tmp/a exist. +fs.mkdir('/tmp/a/apple', { recursive: true }, (err) => { + if (err) throw err; +}); +``` + See also: mkdir(2). -## fs.mkdirSync(path[, mode]) +## fs.mkdirSync(path[, options]) * `path` {string|Buffer|URL} -* `mode` {integer} Not supported on Windows. **Default:** `0o777`. +* `options` {Object|integer} + * `recursive` {boolean} **Default:** `false` + * `mode` {integer} Not supported on Windows. **Default:** `0o777`. Synchronously creates a directory. Returns `undefined`. This is the synchronous version of [`fs.mkdir()`][]. @@ -3979,18 +3994,24 @@ changes: Asynchronous lstat(2). The `Promise` is resolved with the [`fs.Stats`][] object for the given symbolic link `path`. -### fsPromises.mkdir(path[, mode]) +### fsPromises.mkdir(path[, options]) * `path` {string|Buffer|URL} -* `mode` {integer} **Default:** `0o777` +* `options` {Object|integer} + * `recursive` {boolean} **Default:** `false` + * `mode` {integer} Not supported on Windows. **Default:** `0o777`. * Returns: {Promise} Asynchronously creates a directory then resolves the `Promise` with no arguments upon success. +The optional `options` argument can be an integer specifying mode (permission +and sticky bits), or an object with a `mode` property and a `recursive` +property indicating whether parent folders should be created. + ### fsPromises.mkdtemp(prefix[, options]) ' % ( + render_traceback(self, full=full), + self.render_as_text().decode('utf-8', 'replace') + ) + + @property + def is_template_syntax_error(self): + """`True` if this is a template syntax error.""" + return isinstance(self.exc_value, TemplateSyntaxError) + + @property + def exc_info(self): + """Exception info tuple with a proxy around the frame objects.""" + return self.exc_type, self.exc_value, self.frames[0] + + @property + def standard_exc_info(self): + """Standard python exc_info for re-raising""" + tb = self.frames[0] + # the frame will be an actual traceback (or transparent proxy) if + # we are on pypy or a python implementation with support for tproxy + if type(tb) is not TracebackType: + tb = tb.tb + return self.exc_type, self.exc_value, tb + + +def make_traceback(exc_info, source_hint=None): + """Creates a processed traceback object from the exc_info.""" + exc_type, exc_value, tb = exc_info + if isinstance(exc_value, TemplateSyntaxError): + exc_info = translate_syntax_error(exc_value, source_hint) + initial_skip = 0 + else: + initial_skip = 1 + return translate_exception(exc_info, initial_skip) + + +def translate_syntax_error(error, source=None): + """Rewrites a syntax error to please traceback systems.""" + error.source = source + error.translated = True + exc_info = (error.__class__, error, None) + filename = error.filename + if filename is None: + filename = '' + return fake_exc_info(exc_info, filename, error.lineno) + + +def translate_exception(exc_info, initial_skip=0): + """If passed an exc_info it will automatically rewrite the exceptions + all the way down to the correct line numbers and frames. + """ + tb = exc_info[2] + frames = [] + + # skip some internal frames if wanted + for x in range(initial_skip): + if tb is not None: + tb = tb.tb_next + initial_tb = tb + + while tb is not None: + # skip frames decorated with @internalcode. These are internal + # calls we can't avoid and that are useless in template debugging + # output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + # save a reference to the next frame if we override the current + # one with a faked one. + next = tb.tb_next + + # fake template exceptions + template = tb.tb_frame.f_globals.get('__jinja_template__') + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, + lineno)[2] + + frames.append(make_frame_proxy(tb)) + tb = next + + # if we don't have any exceptions in the frames left, we have to + # reraise it unchanged. + # XXX: can we backup here? when could this happen? + if not frames: + reraise(exc_info[0], exc_info[1], exc_info[2]) + + return ProcessedTraceback(exc_info[0], exc_info[1], frames) + + +def get_jinja_locals(real_locals): + ctx = real_locals.get('context') + if ctx: + locals = ctx.get_all().copy() + else: + locals = {} + + local_overrides = {} + + for name, value in iteritems(real_locals): + if not name.startswith('l_') or value is missing: + continue + try: + _, depth, name = name.split('_', 2) + depth = int(depth) + except ValueError: + continue + cur_depth = local_overrides.get(name, (-1,))[0] + if cur_depth < depth: + local_overrides[name] = (depth, value) + + for name, (_, value) in iteritems(local_overrides): + if value is missing: + locals.pop(name, None) + else: + locals[name] = value + + return locals + + +def fake_exc_info(exc_info, filename, lineno): + """Helper for `translate_exception`.""" + exc_type, exc_value, tb = exc_info + + # figure the real context out + if tb is not None: + locals = get_jinja_locals(tb.tb_frame.f_locals) + + # if there is a local called __jinja_exception__, we get + # rid of it to not break the debug functionality. + locals.pop('__jinja_exception__', None) + else: + locals = {} + + # assamble fake globals we need + globals = { + '__name__': filename, + '__file__': filename, + '__jinja_exception__': exc_info[:2], + + # we don't want to keep the reference to the template around + # to not cause circular dependencies, but we mark it as Jinja + # frame for the ProcessedTraceback + '__jinja_template__': None + } + + # and fake the exception + code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') + + # if it's possible, change the name of the code. This won't work + # on some python environments such as google appengine + try: + if tb is None: + location = 'template' + else: + function = tb.tb_frame.f_code.co_name + if function == 'root': + location = 'top-level template code' + elif function.startswith('block_'): + location = 'block "%s"' % function[6:] + else: + location = 'template' + + if PY2: + code = CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + else: + code = CodeType(0, code.co_kwonlyargcount, + code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + except Exception as e: + pass + + # execute the code and catch the new traceback + try: + exec(code, globals, locals) + except: + exc_info = sys.exc_info() + new_tb = exc_info[2].tb_next + + # return without this frame + return exc_info[:2] + (new_tb,) + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. Do not attempt to use this on non cpython + interpreters + """ + import ctypes + from types import TracebackType + + if PY2: + # figure out size of _Py_ssize_t for Python 2: + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + else: + # platform ssize_t on Python 3 + _Py_ssize_t = ctypes.c_ssize_t + + # regular python + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if hasattr(sys, 'getobjects'): + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and + (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +# try to get a tb_set_next implementation if we don't have transparent +# proxies. +tb_set_next = None +if tproxy is None: + try: + tb_set_next = _init_ugly_crap() + except: + pass + del _init_ugly_crap diff --git a/tools/jinja2/defaults.py b/tools/jinja2/defaults.py new file mode 100644 index 000000000000..7c93dec0aeb2 --- /dev/null +++ b/tools/jinja2/defaults.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" + jinja2.defaults + ~~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from jinja2._compat import range_type +from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace + + +# defaults for the parser / lexer +BLOCK_START_STRING = '{%' +BLOCK_END_STRING = '%}' +VARIABLE_START_STRING = '{{' +VARIABLE_END_STRING = '}}' +COMMENT_START_STRING = '{#' +COMMENT_END_STRING = '#}' +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False + + +# default filters, tests and namespace +from jinja2.filters import FILTERS as DEFAULT_FILTERS +from jinja2.tests import TESTS as DEFAULT_TESTS +DEFAULT_NAMESPACE = { + 'range': range_type, + 'dict': dict, + 'lipsum': generate_lorem_ipsum, + 'cycler': Cycler, + 'joiner': Joiner, + 'namespace': Namespace +} + + +# default policies +DEFAULT_POLICIES = { + 'compiler.ascii_str': True, + 'urlize.rel': 'noopener', + 'urlize.target': None, + 'truncate.leeway': 5, + 'json.dumps_function': None, + 'json.dumps_kwargs': {'sort_keys': True}, + 'ext.i18n.trimmed': False, +} + + +# export all constants +__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/tools/jinja2/environment.py b/tools/jinja2/environment.py new file mode 100644 index 000000000000..549d9afab456 --- /dev/null +++ b/tools/jinja2/environment.py @@ -0,0 +1,1276 @@ +# -*- coding: utf-8 -*- +""" + jinja2.environment + ~~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import sys +import weakref +from functools import reduce, partial +from jinja2 import nodes +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ + DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS +from jinja2.lexer import get_lexer, TokenStream +from jinja2.parser import Parser +from jinja2.nodes import EvalContext +from jinja2.compiler import generate, CodeGenerator +from jinja2.runtime import Undefined, new_context, Context +from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ + TemplatesNotFound, TemplateRuntimeError +from jinja2.utils import import_string, LRUCache, Markup, missing, \ + concat, consume, internalcode, have_async_gen +from jinja2._compat import imap, ifilter, string_types, iteritems, \ + text_type, reraise, implements_iterator, implements_to_string, \ + encode_filename, PY2, PYPY + + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + +# the function to create jinja traceback objects. This is dynamically +# imported on the first exception in the exception handler. +_make_traceback = None + + +def get_spontaneous_environment(*args): + """Return a new spontaneous environment. A spontaneous environment is an + unnamed and unaccessible (in theory) environment that is used for + templates generated from a string and not from the file system. + """ + try: + env = _spontaneous_environments.get(args) + except TypeError: + return Environment(*args) + if env is not None: + return env + _spontaneous_environments[args] = env = Environment(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(string, name): + msg = string % name + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e) + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass(environment.undefined, Undefined), 'undefined must ' \ + 'be a subclass of undefined because filters depend on it.' + assert environment.block_start_string != \ + environment.variable_start_string != \ + environment.comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' + assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ + 'newline_sequence set to unknown line ending string.' + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which allows + you to take advantage of newer Python features. This requires + Python 3.6 or later. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: these are currently EXPERIMENTAL undocumented features. + exception_handler = None + exception_formatter = None + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__(self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay(self, block_start_string=missing, block_end_string=missing, + variable_start_string=missing, variable_end_string=missing, + comment_start_string=missing, comment_end_string=missing, + line_statement_prefix=missing, line_comment_prefix=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, + undefined=missing, finalize=missing, autoescape=missing, + loader=missing, cache_size=missing, auto_reload=missing, + bytecode_cache=missing): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args['self'], args['cache_size'], args['extensions'] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), + key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + Note that on Python 3 this might return a coroutine in case the + filter is running from an environment in async mode and the filter + supports async execution. It's your responsibility to await this + if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable('no filter named %r', name) + args = [value] + list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable('no test named %r', name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja2 extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce(lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), text_type(source)) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate(source, self, name, filename, defer_init=defer_init, + optimized=self.optimized) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, 'exec') + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, + defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, + defer_init=defer_init) + if raw: + return source + if filename is None: + filename = '