From 33a373ba41d978af60c2f0230bcba6ad27357ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20M=C3=BCller?= Date: Wed, 29 Jan 2020 16:25:11 +0100 Subject: [PATCH] Port build to python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * first tool of choice was 2to3 * some parts were taken from [1] but during work it was found that this patch introduces interesting effects - see hash functions. Working more on this makes me guess that one has never worked... * Few parts were taken from upstream mirror [2]. Since they use six for porting to python3 it adds us a new dependency. * To get a better overview what is going on or failing some additional messages were added. The most verbose one is left disabled - see python/mozbuild/mozbuild/configure/__init__.py / Line 310 onwards * major changes upstream on build are not to expect so upgrading should cause little trouble and changes can be tracked by [3] * some solutions are workarounds/hacks so this patch will not be accepeted upstream. This should not be a probelme for us: once mozjs >= 68 will arrive we have to go to rust/cargo based build anyway. [1] https://code.foxkit.us/adelie/packages/blob/f2b5773da19ab397fbe64fd32dacc383cfe4cd77/user/mozjs/python3.patch [2] https://github.com/mozilla/gecko-dev [3] https://github.com/mozilla/gecko-dev/tree/esr60 Upstream-Status: Inaproppriate [Some Hacks] Signed-off-by: Andreas Müller --- build/autoconf/config.status.m4 | 2 +- build/moz.configure/android-ndk.configure | 4 +- build/moz.configure/checks.configure | 4 +- build/moz.configure/init.configure | 31 +- build/moz.configure/keyfiles.configure | 4 +- build/moz.configure/old.configure | 32 +- build/moz.configure/toolchain.configure | 18 +- build/moz.configure/util.configure | 9 +- build/moz.configure/windows.configure | 10 +- build/templates.mozbuild | 2 +- config/MozZipFile.py | 12 +- config/expandlibs.py | 6 +- config/expandlibs_exec.py | 14 +- config/expandlibs_gen.py | 4 +- configure.py | 42 +- js/src/build/moz.build | 8 +- js/src/builtin/embedjs.py | 10 +- js/src/configure | 2 +- js/src/frontend/GenerateReservedWords.py | 6 +- js/src/gc/GenerateStatsPhases.py | 4 +- js/src/old-configure.in | 2 + memory/build/moz.build | 8 +- mozglue/build/moz.build | 22 +- .../mozbuild/mozbuild/action/check_binary.py | 2 + .../mozbuild/action/process_define_files.py | 4 +- python/mozbuild/mozbuild/backend/base.py | 8 +- python/mozbuild/mozbuild/backend/common.py | 8 +- .../mozbuild/backend/configenvironment.py | 14 +- .../mozbuild/mozbuild/backend/fastermake.py | 10 +- .../mozbuild/backend/recursivemake.py | 181 +++---- python/mozbuild/mozbuild/config_status.py | 7 +- .../mozbuild/mozbuild/configure/__init__.py | 83 +++- .../mozbuild/configure/check_debug_ranges.py | 6 +- python/mozbuild/mozbuild/configure/options.py | 24 +- python/mozbuild/mozbuild/configure/util.py | 12 +- .../mozbuild/mozbuild/controller/building.py | 16 +- python/mozbuild/mozbuild/frontend/context.py | 89 ++-- python/mozbuild/mozbuild/frontend/data.py | 8 +- python/mozbuild/mozbuild/frontend/emitter.py | 50 +- python/mozbuild/mozbuild/frontend/reader.py | 49 +- python/mozbuild/mozbuild/frontend/sandbox.py | 3 +- python/mozbuild/mozbuild/jar.py | 12 +- python/mozbuild/mozbuild/makeutil.py | 24 +- python/mozbuild/mozbuild/mozinfo.py | 8 +- python/mozbuild/mozbuild/preprocessor.py | 27 +- python/mozbuild/mozbuild/shellutil.py | 6 +- .../test/backend/test_recursivemake.py | 18 +- .../mozbuild/test/configure/common.py | 8 +- .../mozbuild/mozbuild/test/configure/lint.py | 8 +- .../test/configure/test_checks_configure.py | 8 +- .../test/configure/test_compile_checks.py | 4 +- .../mozbuild/test/configure/test_configure.py | 244 +++++----- .../mozbuild/test/configure/test_lint.py | 24 +- .../test/configure/test_moz_configure.py | 32 +- .../mozbuild/test/configure/test_options.py | 450 +++++++++--------- .../configure/test_toolchain_configure.py | 22 +- .../test/configure/test_toolchain_helpers.py | 62 +-- .../configure/test_toolkit_moz_configure.py | 2 +- .../mozbuild/test/configure/test_util.py | 8 +- python/mozbuild/mozbuild/testing.py | 10 +- python/mozbuild/mozbuild/util.py | 79 ++- python/mozbuild/mozbuild/virtualenv.py | 6 +- python/mozbuild/mozpack/chrome/manifest.py | 6 +- python/mozbuild/mozpack/copier.py | 12 +- python/mozbuild/mozpack/files.py | 22 +- python/mozbuild/mozpack/manifests.py | 16 +- python/mozbuild/mozpack/mozjar.py | 37 +- .../manifestparser/manifestparser/ini.py | 13 +- .../manifestparser/manifestparser.py | 24 +- testing/mozbase/mozinfo/mozinfo/mozinfo.py | 26 +- .../mozprocess/mozprocess/processhandler.py | 10 +- third_party/python/which/which.py | 18 +- 72 files changed, 1081 insertions(+), 993 deletions(-) diff --git a/build/autoconf/config.status.m4 b/build/autoconf/config.status.m4 index c75575386..543c2d682 100644 --- a/build/autoconf/config.status.m4 +++ b/build/autoconf/config.status.m4 @@ -122,7 +122,7 @@ trap 'rm -f $CONFIG_STATUS conftest*; exit 1' 1 2 15 dnl We're going to need [ ] for python syntax. changequote(<<<, >>>)dnl -echo creating $CONFIG_STATUS +echo creating $CONFIG_STATUS in `pwd` cat > $CONFIG_STATUS <./config.log', '>>./config.log') + script = script.replace(b'>./config.log', b'>>./config.log') with open(old_configure, 'wb') as fh: fh.write(script) @@ -282,8 +282,8 @@ def old_configure_options(*options): '--x-includes', '--x-libraries', ) -@imports(_from='__builtin__', _import='compile') -@imports(_from='__builtin__', _import='open') +@imports(_from='builtins', _import='compile') +@imports(_from='builtins', _import='open') @imports('logging') @imports('os') @imports('subprocess') @@ -326,7 +326,7 @@ def old_configure(prepare_configure, extra_old_configure_args, all_options, log.debug('Running %s', quote(*cmd)) if extra_env: log.debug('with extra environment: %s', - ' '.join('%s=%s' % pair for pair in extra_env.iteritems())) + ' '.join('%s=%s' % pair for pair in extra_env.items())) # Our logging goes to config.log, the same file old.configure uses. # We can't share the handle on the file, so close it. We assume nothing @@ -359,7 +359,7 @@ def old_configure(prepare_configure, extra_old_configure_args, all_options, # Every variation of the exec() function I tried led to: # SyntaxError: unqualified exec is not allowed in function 'main' it # contains a nested function with free variables - exec code in raw_config # noqa + exec(code, raw_config) # noqa # Ensure all the flags known to old-configure appear in the # @old_configure_options above. @@ -393,16 +393,24 @@ def set_old_configure_define(name, value): @depends(old_configure) @imports('types') def post_old_configure(raw_config): + log.info('post_old_configure started') + for k, v in raw_config['substs']: set_old_configure_config( - k[1:-1], v[1:-1] if isinstance(v, types.StringTypes) else v) + k[1:-1], v[1:-1] if isinstance(v, str) else v) + + log.info('post_old_configure 1 finished') - for k, v in dict(raw_config['defines']).iteritems(): + for k, v in dict(raw_config['defines']).items(): set_old_configure_define(k[1:-1], v[1:-1]) + log.info('post_old_configure 2 finished') + set_old_configure_config('non_global_defines', raw_config['non_global_defines']) + log.info('post_old_configure 3 finished') + # Assuming no other option is declared after this function, handle the # env options that were injected by mozconfig_options by creating dummy @@ -414,6 +422,7 @@ def post_old_configure(raw_config): @imports('__sandbox__') @imports(_from='mozbuild.configure.options', _import='Option') def remaining_mozconfig_options(_): + log.info('remaining_mozconfig_options started') helper = __sandbox__._helper for arg in helper: if helper._origins[arg] != 'mozconfig': @@ -422,5 +431,6 @@ def remaining_mozconfig_options(_): if name.isupper() and name not in __sandbox__._options: option = Option(env=name, nargs='*', help=name) helper.handle(option) + log.info('remaining_mozconfig_options finished') # Please do not add anything after remaining_mozconfig_options() diff --git a/build/moz.configure/toolchain.configure b/build/moz.configure/toolchain.configure index fc640c75e..c5508dfb7 100755 --- a/build/moz.configure/toolchain.configure +++ b/build/moz.configure/toolchain.configure @@ -391,7 +391,7 @@ def get_compiler_info(compiler, language): ('CPU', CPU_preprocessor_checks), ('KERNEL', kernel_preprocessor_checks), ): - for n, (value, condition) in enumerate(preprocessor_checks.iteritems()): + for n, (value, condition) in enumerate(preprocessor_checks.items()): check += dedent('''\ #%(if)s %(condition)s %%%(name)s "%(value)s" @@ -425,9 +425,9 @@ def get_compiler_info(compiler, language): data = {} for line in result.splitlines(): if line.startswith(b'%'): - k, _, v = line.partition(' ') - k = k.lstrip('%') - data[k] = v.replace(' ', '').lstrip('"').rstrip('"') + k, _, v = line.partition(b' ') + k = k.lstrip(b'%').decode('utf-8') + data[k] = v.replace(b' ', b'').lstrip(b'"').rstrip(b'"').decode('utf-8') log.debug('%s = %s', k, data[k]) try: @@ -551,7 +551,7 @@ def check_compiler(compiler, language, target): ) -@imports(_from='__builtin__', _import='open') +@imports(_from='builtins', _import='open') @imports('json') @imports('subprocess') @imports('sys') @@ -606,7 +606,7 @@ def vs_major_version(value): @depends(host, target, vs_major_version, check_build_environment, '--with-visual-studio-version') -@imports(_from='__builtin__', _import='sorted') +@imports(_from='builtins', _import='sorted') @imports(_from='operator', _import='itemgetter') @imports('platform') def vc_compiler_path(host, target, vs_major_version, env, vs_release_name): @@ -807,7 +807,7 @@ def compiler(language, host_or_target, c_compiler=None, other_compiler=None, target.os != 'Android': return namespace(**{ k: [] if k == 'flags' else v - for k, v in other_compiler.__dict__.iteritems() + for k, v in other_compiler.__dict__.items() }) # Normally, we'd use `var` instead of `_var`, but the interaction with @@ -1238,7 +1238,7 @@ set_config('VISIBILITY_FLAGS', visibility_flags) @depends(c_compiler) @imports('multiprocessing') -@imports(_from='__builtin__', _import='min') +@imports(_from='builtins', _import='min') def pgo_flags(compiler): if compiler.type in ('gcc', 'clang'): return namespace( @@ -1517,6 +1517,8 @@ def enable_gnu_linker(enable_gold_option, c_compiler, developer_options, build_e gold = check_cmd_output(*detection_cmd).strip() if not gold: return + if isinstance(gold, bytes): + gold = gold.decode('utf-8') goldFullPath = find_program(gold) if goldFullPath is None: diff --git a/build/moz.configure/util.configure b/build/moz.configure/util.configure index 3284fd8b5..218813e2d 100644 --- a/build/moz.configure/util.configure +++ b/build/moz.configure/util.configure @@ -25,7 +25,6 @@ def configure_error(message): # does not. -@imports(_from='__builtin__', _import='unicode') @imports('subprocess') @imports('sys') @imports(_from='mozbuild.configure.util', _import='LineIO') @@ -39,10 +38,10 @@ def check_cmd_output(*args, **kwargs): if 'env' in kwargs: normalized_env = {} for k, v in kwargs['env'].items(): - if isinstance(k, unicode): + if isinstance(k, str): k = k.encode('utf-8', 'strict') - if isinstance(v, unicode): + if isinstance(v, str): v = v.encode('utf-8', 'strict') normalized_env[k] = v @@ -285,7 +284,7 @@ def unique_list(l): # ('19.0', 'x64', r'C:\...\amd64\cl.exe') # ('19.0', 'x86', r'C:\...\amd64_x86\cl.exe') @imports(_import='_winreg', _as='winreg') -@imports(_from='__builtin__', _import='WindowsError') +@imports(_from='builtins', _import='WindowsError') @imports(_from='fnmatch', _import='fnmatch') def get_registry_values(pattern, get_32_and_64_bit=False): def enum_helper(func, key): @@ -360,6 +359,8 @@ def get_registry_values(pattern, get_32_and_64_bit=False): @imports(_from='mozbuild.configure.util', _import='Version', _as='_Version') def Version(v): 'A version number that can be compared usefully.' + if isinstance(v, bytes): + v = v.decode('utf-8') return _Version(v) # Denotes a deprecated option. Combines option() and @depends: diff --git a/build/moz.configure/windows.configure b/build/moz.configure/windows.configure index a5b790e3b..2b88fc447 100644 --- a/build/moz.configure/windows.configure +++ b/build/moz.configure/windows.configure @@ -10,7 +10,7 @@ option('--with-windows-version', nargs=1, default='603', @depends('--with-windows-version') -@imports(_from='__builtin__', _import='ValueError') +@imports(_from='builtins', _import='ValueError') def valid_windows_version(value): if not value: die('Cannot build with --without-windows-version') @@ -50,8 +50,8 @@ def windows_sdk_dir(value, host): @imports('os') @imports('re') -@imports(_from='__builtin__', _import='sorted') -@imports(_from='__builtin__', _import='WindowsError') +@imports(_from='builtins', _import='sorted') +@imports(_from='builtins', _import='WindowsError') def get_sdk_dirs(sdk, subdir): def get_dirs_containing(sdk, stem, subdir): base = os.path.join(sdk, stem) @@ -96,7 +96,7 @@ def valid_windows_sdk_dir_result(value): @depends(c_compiler, windows_sdk_dir, valid_windows_version, 'WINDOWSSDKDIR') @checking('for Windows SDK', valid_windows_sdk_dir_result) -@imports(_from='__builtin__', _import='sorted') +@imports(_from='builtins', _import='sorted') @imports(_from='textwrap', _import='dedent') def valid_windows_sdk_dir(compiler, windows_sdk_dir, target_version, windows_sdk_dir_env): @@ -174,7 +174,7 @@ def valid_ucrt_sdk_dir_result(value): @depends(windows_sdk_dir, 'WINDOWSSDKDIR', c_compiler) @checking('for Universal CRT SDK', valid_ucrt_sdk_dir_result) @imports('os') -@imports(_from='__builtin__', _import='sorted') +@imports(_from='builtins', _import='sorted') @imports(_import='mozpack.path', _as='mozpath') def valid_ucrt_sdk_dir(windows_sdk_dir, windows_sdk_dir_env, c_compiler): if windows_sdk_dir_env: diff --git a/build/templates.mozbuild b/build/templates.mozbuild index 3da850ce5..ae5e410fe 100644 --- a/build/templates.mozbuild +++ b/build/templates.mozbuild @@ -10,7 +10,7 @@ def Binary(): templates.''' # Add -llog by default, since we use it all over the place. - if CONFIG['OS_TARGET'] == 'Android': + if str(CONFIG['OS_TARGET']) == 'Android': OS_LIBS += ['log'] diff --git a/config/MozZipFile.py b/config/MozZipFile.py index 337fe0521..dc7add4c3 100644 --- a/config/MozZipFile.py +++ b/config/MozZipFile.py @@ -18,7 +18,7 @@ class ZipFile(zipfile.ZipFile): def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED, lock = False): if lock: - assert isinstance(file, basestring) + assert isinstance(file, str) self.lockfile = lock_file(file + '.lck') else: self.lockfile = None @@ -46,7 +46,7 @@ class ZipFile(zipfile.ZipFile): date_time=time.localtime(time.time())) zinfo.compress_type = self.compression # Add some standard UNIX file access permissions (-rw-r--r--). - zinfo.external_attr = (0x81a4 & 0xFFFF) << 16L + zinfo.external_attr = (0x81a4 & 0xFFFF) << 16 else: zinfo = zinfo_or_arcname @@ -58,7 +58,7 @@ class ZipFile(zipfile.ZipFile): # as the old, reuse the existing entry. doSeek = False # store if we need to seek to the eof after overwriting - if self.NameToInfo.has_key(zinfo.filename): + if zinfo.filename in self.NameToInfo: # Find the last ZipInfo with our name. # Last, because that's catching multiple overwrites i = len(self.filelist) @@ -109,14 +109,14 @@ class ZipFile(zipfile.ZipFile): # adjust file mode if we originally just wrote, now we rewrite self.fp.close() self.fp = open(self.filename, 'r+b') - all = map(lambda zi: (zi, True), self.filelist) + \ - map(lambda zi: (zi, False), self._remove) + all = [(zi, True) for zi in self.filelist] + \ + [(zi, False) for zi in self._remove] all.sort(lambda l, r: cmp(l[0].header_offset, r[0].header_offset)) # empty _remove for multiple closes self._remove = [] lengths = [all[i+1][0].header_offset - all[i][0].header_offset - for i in xrange(len(all)-1)] + for i in range(len(all)-1)] lengths.append(self.end - all[-1][0].header_offset) to_pos = 0 for (zi, keep), length in zip(all, lengths): diff --git a/config/expandlibs.py b/config/expandlibs.py index ac06c432f..df1fed15d 100644 --- a/config/expandlibs.py +++ b/config/expandlibs.py @@ -26,7 +26,7 @@ ${LIB_PREFIX}${ROOT}.${LIB_SUFFIX} following these rules: descriptor contains. And for each of these LIBS, also apply the same rules. ''' -from __future__ import with_statement + import sys, os, errno import expandlibs_config as conf @@ -36,7 +36,7 @@ def ensureParentDir(file): if dir and not os.path.exists(dir): try: os.makedirs(dir) - except OSError, error: + except OSError as error: if error.errno != errno.EEXIST: raise @@ -140,4 +140,4 @@ class ExpandArgs(list): return [relativize(arg)] if __name__ == '__main__': - print " ".join(ExpandArgs(sys.argv[1:])) + print(" ".join(ExpandArgs(sys.argv[1:]))) diff --git a/config/expandlibs_exec.py b/config/expandlibs_exec.py index df656016c..fb786a6a8 100644 --- a/config/expandlibs_exec.py +++ b/config/expandlibs_exec.py @@ -20,7 +20,7 @@ With the --symbol-order argument, followed by a file name, it will add the relevant linker options to change the order in which the linker puts the symbols appear in the resulting binary. Only works for ELF targets. ''' -from __future__ import with_statement + import sys import os from expandlibs import ( @@ -304,11 +304,11 @@ class SectionFinder(object): return syms def print_command(out, args): - print >>out, "Executing: " + " ".join(args) + print("Executing: " + " ".join(args), file=out) for tmp in [f for f in args.tmp if os.path.isfile(f)]: - print >>out, tmp + ":" + print(tmp + ":", file=out) with open(tmp) as file: - print >>out, "".join([" " + l for l in file.readlines()]) + print("".join([" " + l for l in file.readlines()]), file=out) out.flush() def main(args, proc_callback=None): @@ -338,13 +338,13 @@ def main(args, proc_callback=None): proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) if proc_callback: proc_callback(proc) - except Exception, e: - print >>sys.stderr, 'error: Launching', args, ':', e + except Exception as e: + print('error: Launching', args, ':', e, file=sys.stderr) raise e (stdout, stderr) = proc.communicate() if proc.returncode and not options.verbose: print_command(sys.stderr, args) - sys.stderr.write(stdout) + sys.stderr.write(stdout.decode("utf-8")) sys.stderr.flush() if proc.returncode: return proc.returncode diff --git a/config/expandlibs_gen.py b/config/expandlibs_gen.py index b1de63cd0..dc62bd184 100644 --- a/config/expandlibs_gen.py +++ b/config/expandlibs_gen.py @@ -5,7 +5,7 @@ '''Given a list of object files and library names, prints a library descriptor to standard output''' -from __future__ import with_statement + import sys import os import expandlibs_config as conf @@ -38,4 +38,4 @@ if __name__ == '__main__': ensureParentDir(options.output) with open(options.output, 'w') as outfile: - print >>outfile, generate(args) + print(generate(args), file=outfile) diff --git a/configure.py b/configure.py index 771e34e38..bee329d7c 100644 --- a/configure.py +++ b/configure.py @@ -2,10 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function, unicode_literals + import codecs import itertools +import logging import os import sys import textwrap @@ -34,7 +35,9 @@ from mozbuild.util import ( def main(argv): config = {} sandbox = ConfigureSandbox(config, os.environ, argv) + print('sandbox.run started') sandbox.run(os.path.join(os.path.dirname(__file__), 'moz.configure')) + print('sandbox.run finished') if sandbox._help: return 0 @@ -56,12 +59,21 @@ def config_status(config): sanitized_config = {} sanitized_config['substs'] = { - k: sanitized_bools(v) for k, v in config.iteritems() + k: sanitized_bools(v) for k, v in config.items() if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR', 'ALL_CONFIGURE_PATHS') } + + # Hack around OptionValue entries unknown during compile + for opt in ('BUILD_BACKENDS', 'MOZ_UI_LOCALE', 'RUSTFLAGS' ): + old = sanitized_config['substs'][opt] + new = [] + for setting in old: + new.append(setting) + sanitized_config['substs'][opt] = new + sanitized_config['defines'] = { - k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems() + k: sanitized_bools(v) for k, v in config['DEFINES'].items() } sanitized_config['non_global_defines'] = config['non_global_defines'] sanitized_config['topsrcdir'] = config['TOPSRCDIR'] @@ -71,20 +83,17 @@ def config_status(config): # Create config.status. Eventually, we'll want to just do the work it does # here, when we're able to skip configure tests/use cached results/not rely # on autoconf. - print("Creating config.status", file=sys.stderr) - encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8' - with codecs.open('config.status', 'w', encoding) as fh: + logging.getLogger('moz.configure').info('Creating config.status') + with codecs.open('config.status', 'w', 'utf-8') as fh: fh.write(textwrap.dedent('''\ #!%(python)s - # coding=%(encoding)s - from __future__ import unicode_literals - from mozbuild.util import encode - encoding = '%(encoding)s' - ''') % {'python': config['PYTHON'], 'encoding': encoding}) + # coding=utf-8 + print("config.status started") + ''') % {'python': config['PYTHON']}) # A lot of the build backend code is currently expecting byte # strings and breaks in subtle ways with unicode strings. (bug 1296508) - for k, v in sanitized_config.iteritems(): - fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v))) + for k, v in sanitized_config.items(): + fh.write('%s = %s\n' % (k, indented_repr(v))) fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'non_global_defines', 'substs', 'mozconfig']") @@ -97,6 +106,9 @@ def config_status(config): args = dict([(name, globals()[name]) for name in __all__]) config_status(**args) ''')) + fh.write(textwrap.dedent(''' + print("config.status finished") + ''')) partial_config = PartialConfigEnvironment(config['TOPOBJDIR']) partial_config.write_vars(sanitized_config) @@ -116,7 +128,7 @@ def config_status(config): # executable permissions. os.chmod('config.status', 0o755) if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'): - os.environ[b'WRITE_MOZINFO'] = b'1' + os.environ['WRITE_MOZINFO'] = '1' from mozbuild.config_status import config_status # Some values in sanitized_config also have more complex types, such as @@ -127,7 +139,7 @@ def config_status(config): # A lot of the build backend code is currently expecting byte strings # and breaks in subtle ways with unicode strings. - return config_status(args=[], **encode(sanitized_config, encoding)) + return config_status(args=[], **sanitized_config) return 0 diff --git a/js/src/build/moz.build b/js/src/build/moz.build index a7f5fa4ce..856cae32d 100644 --- a/js/src/build/moz.build +++ b/js/src/build/moz.build @@ -47,22 +47,22 @@ USE_LIBS += [ 'zlib', ] -if CONFIG['OS_ARCH'] not in ('WINNT', 'HP-UX'): +if str(CONFIG['OS_ARCH']) not in ('WINNT', 'HP-UX'): OS_LIBS += [ 'm', ] -if CONFIG['OS_ARCH'] == 'FreeBSD': +if str(CONFIG['OS_ARCH']) == 'FreeBSD': OS_LIBS += [ '-pthread', ] -if CONFIG['OS_ARCH'] == 'Linux': +if str(CONFIG['OS_ARCH']) == 'Linux': OS_LIBS += [ 'dl', ] -if CONFIG['OS_ARCH'] == 'SunOS': +if str(CONFIG['OS_ARCH']) == 'SunOS': OS_LIBS += [ 'posix4', 'dl', diff --git a/js/src/builtin/embedjs.py b/js/src/builtin/embedjs.py index ba25e71c1..d4f2de122 100644 --- a/js/src/builtin/embedjs.py +++ b/js/src/builtin/embedjs.py @@ -36,7 +36,7 @@ # # It uses the C preprocessor to process its inputs. -from __future__ import with_statement + import re, sys, os, subprocess import shlex import which @@ -52,8 +52,8 @@ def ToCAsciiArray(lines): def ToCArray(lines): result = [] - for chr in lines: - result.append(str(ord(chr))) + for char in lines: + result.append("0x%0.2X" % char) return ", ".join(result) HEADER_TEMPLATE = """\ @@ -87,7 +87,7 @@ def embed(cxx, preprocessorOption, cppflags, msgs, sources, c_out, js_out, names js_out.write(processed) import zlib - compressed = zlib.compress(processed) + compressed = zlib.compress(processed.encode('utf-8')) data = ToCArray(compressed) c_out.write(HEADER_TEMPLATE % { 'sources_type': 'unsigned char', @@ -107,7 +107,7 @@ def preprocess(cxx, preprocessorOption, source, args = []): tmpOut = 'self-hosting-preprocessed.pp'; outputArg = shlex.split(preprocessorOption + tmpOut) - with open(tmpIn, 'wb') as input: + with open(tmpIn, 'w') as input: input.write(source) print(' '.join(cxx + outputArg + args + [tmpIn])) result = subprocess.Popen(cxx + outputArg + args + [tmpIn]).wait() diff --git a/js/src/configure b/js/src/configure index 3b3a39af3..8f5ea41d0 100755 --- a/js/src/configure +++ b/js/src/configure @@ -24,4 +24,4 @@ export OLD_CONFIGURE="$SRCDIR"/old-configure set -- "$@" --enable-project=js -which python2.7 > /dev/null && exec python2.7 "$TOPSRCDIR/configure.py" "$@" || exec python "$TOPSRCDIR/configure.py" "$@" +which python3 > /dev/null && exec python3 "$TOPSRCDIR/configure.py" "$@" || exec python "$TOPSRCDIR/configure.py" "$@" diff --git a/js/src/frontend/GenerateReservedWords.py b/js/src/frontend/GenerateReservedWords.py index 3aa2307b9..381c8e2b4 100644 --- a/js/src/frontend/GenerateReservedWords.py +++ b/js/src/frontend/GenerateReservedWords.py @@ -80,14 +80,14 @@ def split_list_per_column(reserved_word_list, column): per_column = column_dict.setdefault(word[column], []) per_column.append(item) - return sorted(column_dict.items(), key=lambda (char, word): ord(char)) + return sorted(list(column_dict.items()), key=lambda char_word: ord(char_word[0])) def generate_letter_switch(opt, unprocessed_columns, reserved_word_list, columns=None): assert(len(reserved_word_list) != 0); if not columns: - columns = range(0, unprocessed_columns) + columns = list(range(0, unprocessed_columns)) if len(reserved_word_list) == 1: index, word = reserved_word_list[0] @@ -161,7 +161,7 @@ def split_list_per_length(reserved_word_list): per_length = length_dict.setdefault(len(word), []) per_length.append(item) - return sorted(length_dict.items(), key=lambda (length, word): length) + return sorted(list(length_dict.items()), key=lambda length_word: length_word[0]) def generate_switch(opt, reserved_word_list): assert(len(reserved_word_list) != 0); diff --git a/js/src/gc/GenerateStatsPhases.py b/js/src/gc/GenerateStatsPhases.py index 2daf83555..e39a26a4b 100644 --- a/js/src/gc/GenerateStatsPhases.py +++ b/js/src/gc/GenerateStatsPhases.py @@ -267,7 +267,7 @@ def generateHeader(out): # # Generate PhaseKind enum. # - phaseKindNames = map(lambda phaseKind: phaseKind.name, AllPhaseKinds) + phaseKindNames = [phaseKind.name for phaseKind in AllPhaseKinds] extraPhaseKinds = [ "NONE = LIMIT", "EXPLICIT_SUSPENSION = LIMIT", @@ -279,7 +279,7 @@ def generateHeader(out): # # Generate Phase enum. # - phaseNames = map(lambda phase: phase.name, AllPhases) + phaseNames = [phase.name for phase in AllPhases] extraPhases = [ "NONE = LIMIT", "EXPLICIT_SUSPENSION = LIMIT", diff --git a/js/src/old-configure.in b/js/src/old-configure.in index 11c3d5a2e..389265404 100644 --- a/js/src/old-configure.in +++ b/js/src/old-configure.in @@ -1884,3 +1884,5 @@ if test "$JS_STANDALONE"; then fi rm -fr confdefs* $ac_clean_files +echo confdefs* $ac_clean_files removed +echo "old-configure done" diff --git a/memory/build/moz.build b/memory/build/moz.build index e2c715271..f09ce7935 100644 --- a/memory/build/moz.build +++ b/memory/build/moz.build @@ -30,7 +30,7 @@ else: 'fallback.cpp', ] -if CONFIG['OS_TARGET'] == 'Darwin' and (CONFIG['MOZ_REPLACE_MALLOC'] or +if str(CONFIG['OS_TARGET']) == 'Darwin' and (CONFIG['MOZ_REPLACE_MALLOC'] or CONFIG['MOZ_MEMORY']): SOURCES += [ 'zone.c', @@ -38,15 +38,15 @@ if CONFIG['OS_TARGET'] == 'Darwin' and (CONFIG['MOZ_REPLACE_MALLOC'] or Library('memory') -if CONFIG['OS_TARGET'] == 'Android' and CONFIG['CC_TYPE'] == 'clang': +if str(CONFIG['OS_TARGET']) == 'Android' and str(CONFIG['CC_TYPE']) == 'clang': CXXFLAGS += [ '-Wno-tautological-pointer-compare', ] -if CONFIG['MOZ_BUILD_APP'] != 'memory': +if str(CONFIG['MOZ_BUILD_APP']) != 'memory': FINAL_LIBRARY = 'mozglue' -if CONFIG['CC_TYPE'] in ('msvc', 'clang-cl'): +if str(CONFIG['CC_TYPE']) in ('msvc', 'clang-cl'): CXXFLAGS += ['-wd4273'] # inconsistent dll linkage (bug 558163) if CONFIG['MOZ_REPLACE_MALLOC_STATIC']: diff --git a/mozglue/build/moz.build b/mozglue/build/moz.build index 53758485a..5e9308802 100644 --- a/mozglue/build/moz.build +++ b/mozglue/build/moz.build @@ -9,12 +9,12 @@ # If this is ever changed, update MOZ_SHARED_MOZGLUE in browser/installer/Makefile.in if CONFIG['JS_STANDALONE'] and not CONFIG['MOZ_MEMORY']: Library('mozglue') -elif CONFIG['OS_TARGET'] in ('WINNT', 'Darwin', 'Android'): +elif str(CONFIG['OS_TARGET']) in ('WINNT', 'Darwin', 'Android'): SharedLibrary('mozglue') else: Library('mozglue') -if CONFIG['OS_TARGET'] == 'Android': +if str(CONFIG['OS_TARGET']) == 'Android': SOURCES += [ 'BionicGlue.cpp', ] @@ -24,14 +24,14 @@ if CONFIG['MOZ_ASAN']: 'AsanOptions.cpp', ] -if CONFIG['OS_TARGET'] == 'WINNT': +if str(CONFIG['OS_TARGET']) == 'WINNT': DEFFILE = 'mozglue.def' # We'll break the DLL blocklist if we immediately load user32.dll DELAYLOAD_DLLS += [ 'user32.dll', ] - if CONFIG['CC_TYPE'] == "msvc": + if str(CONFIG['CC_TYPE']) == "msvc": CFLAGS += ['-guard:cf'] CXXFLAGS += ['-guard:cf'] LDFLAGS += ['-guard:cf'] @@ -48,12 +48,12 @@ if CONFIG['MOZ_WIDGET_TOOLKIT']: 'dummy.cpp', ] - if CONFIG['OS_TARGET'] == 'WINNT': + if str(CONFIG['OS_TARGET']) == 'WINNT': LOCAL_INCLUDES += [ '/memory/build', ] - if CONFIG['CC_TYPE'] == "msvc": + if str(CONFIG['CC_TYPE']) == "msvc": SOURCES += ['WindowsCFGStatus.cpp'] SOURCES += [ 'Authenticode.cpp', @@ -85,17 +85,17 @@ if CONFIG['MOZ_WIDGET_TOOLKIT']: 'WindowsDllBlocklist.h', ] - if CONFIG['CPU_ARCH'].startswith('x86'): + if str(CONFIG['CPU_ARCH']).startswith('x86'): SOURCES += [ 'SSE.cpp', ] - if CONFIG['CPU_ARCH'] == 'arm': + if str(CONFIG['CPU_ARCH']) == 'arm': SOURCES += [ 'arm.cpp', ] - if CONFIG['CPU_ARCH'].startswith('mips'): + if str(CONFIG['CPU_ARCH']).startswith('mips'): SOURCES += [ 'mips.cpp', ] @@ -114,7 +114,7 @@ LIBRARY_DEFINES['MOZ_HAS_MOZGLUE'] = True LDFLAGS += CONFIG['MOZ_GLUE_WRAP_LDFLAGS'] -if CONFIG['OS_TARGET'] == 'Darwin': +if str(CONFIG['OS_TARGET']) == 'Darwin': # On OSX 10.10.3, a dead lock happens in some cases involving dynamic # symbol resolution for symbols that jemalloc itself uses. While it # might be possible to find a way to avoid all such symbol resolutions, @@ -124,7 +124,7 @@ if CONFIG['OS_TARGET'] == 'Darwin': # for TLS. LDFLAGS += ['-Wl,-bind_at_load'] -if CONFIG['MOZ_LINKER'] and CONFIG['TARGET_CPU'] == 'arm': +if CONFIG['MOZ_LINKER'] and str(CONFIG['TARGET_CPU']) == 'arm': LDFLAGS += ['-Wl,-version-script,%s/arm-eabi-filter' % SRCDIR] DIST_INSTALL = True diff --git a/python/mozbuild/mozbuild/action/check_binary.py b/python/mozbuild/mozbuild/action/check_binary.py index 5665ef053..b696f73d6 100644 --- a/python/mozbuild/mozbuild/action/check_binary.py +++ b/python/mozbuild/mozbuild/action/check_binary.py @@ -104,6 +104,8 @@ def iter_readelf_symbols(target, binary): def iter_readelf_dynamic(target, binary): for line in get_output(target['readelf'], '-d', binary): + if isinstance(line, bytes): + line=line.decode('utf-8') data = line.split(None, 2) if data and len(data) == 3 and data[0].startswith('0x'): yield data[1].rstrip(')').lstrip('('), data[2] diff --git a/python/mozbuild/mozbuild/action/process_define_files.py b/python/mozbuild/mozbuild/action/process_define_files.py index 563fbb8fa..c3df2869b 100644 --- a/python/mozbuild/mozbuild/action/process_define_files.py +++ b/python/mozbuild/mozbuild/action/process_define_files.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import argparse import os @@ -53,7 +53,7 @@ def process_define_file(output, input): 'CONFIGURE_DEFINE_FILE') defines = '\n'.join(sorted( '#define %s %s' % (name, val) - for name, val in config.defines['ALLDEFINES'].iteritems())) + for name, val in config.defines['ALLDEFINES'].items())) l = l[:m.start('cmd') - 1] \ + defines + l[m.end('name'):] elif cmd == 'define': diff --git a/python/mozbuild/mozbuild/backend/base.py b/python/mozbuild/mozbuild/backend/base.py index a8d5c94e0..7cda63475 100644 --- a/python/mozbuild/mozbuild/backend/base.py +++ b/python/mozbuild/mozbuild/backend/base.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals + from abc import ( ABCMeta, @@ -31,7 +31,7 @@ from .configenvironment import ConfigEnvironment from mozbuild.base import ExecutionSummary -class BuildBackend(LoggingMixin): +class BuildBackend(LoggingMixin, metaclass=ABCMeta): """Abstract base class for build backends. A build backend is merely a consumer of the build configuration (the output @@ -39,8 +39,6 @@ class BuildBackend(LoggingMixin): is the discretion of the specific implementation. """ - __metaclass__ = ABCMeta - def __init__(self, environment): assert isinstance(environment, (ConfigEnvironment, EmptyConfig)) self.populate_logger() @@ -311,7 +309,7 @@ class BuildBackend(LoggingMixin): srcdir = mozpath.dirname(obj.input_path) pp.context.update({ k: ' '.join(v) if isinstance(v, list) else v - for k, v in obj.config.substs.iteritems() + for k, v in obj.config.substs.items() }) pp.context.update( top_srcdir=obj.topsrcdir, diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index d00cbbcaf..f747df446 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals + import json import os @@ -174,7 +174,7 @@ class CommonBackend(BuildBackend): if len(self._idl_manager.idls): self._write_rust_xpidl_summary(self._idl_manager) self._handle_idl_manager(self._idl_manager) - self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values()) + self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in list(self._idl_manager.idls.values())) for config in self._configs: @@ -372,14 +372,14 @@ class CommonBackend(BuildBackend): with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'rt', 'all.rs')) as fh: fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n") - for idl in manager.idls.values(): + for idl in list(manager.idls.values()): fh.write(include_tmpl % ("rt", idl['root'])) fh.write(";\n") with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'bt', 'all.rs')) as fh: fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n") fh.write("&[\n") - for idl in manager.idls.values(): + for idl in list(manager.idls.values()): fh.write(include_tmpl % ("bt", idl['root'])) fh.write(",\n") fh.write("]\n") diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py index 3676a7d18..f0896cea4 100644 --- a/python/mozbuild/mozbuild/backend/configenvironment.py +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -2,14 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + import os import sys import json from collections import Iterable, OrderedDict -from types import StringTypes, ModuleType +from types import ModuleType import mozpack.path as mozpath @@ -22,7 +22,7 @@ from mozbuild.shellutil import quote as shell_quote if sys.version_info.major == 2: - text_type = unicode + text_type = str else: text_type = str @@ -151,7 +151,7 @@ class ConfigEnvironment(object): shell_quote(self.defines[name]).replace('$', '$$')) for name in sorted(global_defines)]) def serialize(name, obj): - if isinstance(obj, StringTypes): + if isinstance(obj, str): return obj if isinstance(obj, Iterable): return ' '.join(obj) @@ -185,8 +185,8 @@ class ConfigEnvironment(object): except UnicodeDecodeError: return v.decode('utf-8', 'replace') - for k, v in self.substs.items(): - if not isinstance(v, StringTypes): + for k, v in list(self.substs.items()): + if not isinstance(v, str): if isinstance(v, Iterable): type(v)(decode(i) for i in v) elif not isinstance(v, text_type): @@ -255,7 +255,7 @@ class PartialConfigDict(object): existing_files = self._load_config_track() new_files = set() - for k, v in values.iteritems(): + for k, v in values.items(): new_files.add(self._write_file(k, v)) for filename in existing_files - new_files: diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py index b029aa10f..b66ade64f 100644 --- a/python/mozbuild/mozbuild/backend/fastermake.py +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals, print_function + from mozbuild.backend.base import PartialBackend from mozbuild.backend.common import CommonBackend @@ -140,7 +140,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): # Add information for chrome manifest generation manifest_targets = [] - for target, entries in self._manifest_entries.iteritems(): + for target, entries in self._manifest_entries.items(): manifest_targets.append(target) install_target = mozpath.basedir(target, install_manifests_bases) self._install_manifests[install_target].add_content( @@ -152,13 +152,13 @@ class FasterMakeBackend(CommonBackend, PartialBackend): % ' '.join(self._install_manifests.keys())) # Add dependencies we infered: - for target, deps in self._dependencies.iteritems(): + for target, deps in self._dependencies.items(): mk.create_rule([target]).add_dependencies( '$(TOPOBJDIR)/%s' % d for d in deps) mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk') - for base, install_manifest in self._install_manifests.iteritems(): + for base, install_manifest in self._install_manifests.items(): with self._write_file( mozpath.join(self.environment.topobjdir, 'faster', 'install_%s' % base.replace('/', '_'))) as fh: @@ -167,7 +167,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): # For artifact builds only, write a single unified manifest for consumption by |mach watch|. if self.environment.is_artifact_build: unified_manifest = InstallManifest() - for base, install_manifest in self._install_manifests.iteritems(): + for base, install_manifest in self._install_manifests.items(): # Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash. assert base.startswith('dist/bin') base = base[len('dist/bin'):] diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index dd9020d62..aa89cc297 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals + import logging import os @@ -12,7 +12,7 @@ from collections import ( defaultdict, namedtuple, ) -from StringIO import StringIO +from io import StringIO from itertools import chain from mozpack.manifests import ( @@ -80,75 +80,76 @@ from ..util import ( ) from ..makeutil import Makefile from mozbuild.shellutil import quote as shell_quote +from functools import reduce MOZBUILD_VARIABLES = [ - b'ASFLAGS', - b'CMSRCS', - b'CMMSRCS', - b'CPP_UNIT_TESTS', - b'DIRS', - b'DIST_INSTALL', - b'EXTRA_DSO_LDOPTS', - b'EXTRA_JS_MODULES', - b'EXTRA_PP_COMPONENTS', - b'EXTRA_PP_JS_MODULES', - b'FORCE_SHARED_LIB', - b'FORCE_STATIC_LIB', - b'FINAL_LIBRARY', - b'HOST_CFLAGS', - b'HOST_CSRCS', - b'HOST_CMMSRCS', - b'HOST_CXXFLAGS', - b'HOST_EXTRA_LIBS', - b'HOST_LIBRARY_NAME', - b'HOST_PROGRAM', - b'HOST_SIMPLE_PROGRAMS', - b'JAR_MANIFEST', - b'JAVA_JAR_TARGETS', - b'LIBRARY_NAME', - b'LIBS', - b'MAKE_FRAMEWORK', - b'MODULE', - b'NO_DIST_INSTALL', - b'NO_EXPAND_LIBS', - b'NO_INTERFACES_MANIFEST', - b'NO_JS_MANIFEST', - b'OS_LIBS', - b'PARALLEL_DIRS', - b'PREF_JS_EXPORTS', - b'PROGRAM', - b'RESOURCE_FILES', - b'SHARED_LIBRARY_LIBS', - b'SHARED_LIBRARY_NAME', - b'SIMPLE_PROGRAMS', - b'SONAME', - b'STATIC_LIBRARY_NAME', - b'TEST_DIRS', - b'TOOL_DIRS', + 'ASFLAGS', + 'CMSRCS', + 'CMMSRCS', + 'CPP_UNIT_TESTS', + 'DIRS', + 'DIST_INSTALL', + 'EXTRA_DSO_LDOPTS', + 'EXTRA_JS_MODULES', + 'EXTRA_PP_COMPONENTS', + 'EXTRA_PP_JS_MODULES', + 'FORCE_SHARED_LIB', + 'FORCE_STATIC_LIB', + 'FINAL_LIBRARY', + 'HOST_CFLAGS', + 'HOST_CSRCS', + 'HOST_CMMSRCS', + 'HOST_CXXFLAGS', + 'HOST_EXTRA_LIBS', + 'HOST_LIBRARY_NAME', + 'HOST_PROGRAM', + 'HOST_SIMPLE_PROGRAMS', + 'JAR_MANIFEST', + 'JAVA_JAR_TARGETS', + 'LIBRARY_NAME', + 'LIBS', + 'MAKE_FRAMEWORK', + 'MODULE', + 'NO_DIST_INSTALL', + 'NO_EXPAND_LIBS', + 'NO_INTERFACES_MANIFEST', + 'NO_JS_MANIFEST', + 'OS_LIBS', + 'PARALLEL_DIRS', + 'PREF_JS_EXPORTS', + 'PROGRAM', + 'RESOURCE_FILES', + 'SHARED_LIBRARY_LIBS', + 'SHARED_LIBRARY_NAME', + 'SIMPLE_PROGRAMS', + 'SONAME', + 'STATIC_LIBRARY_NAME', + 'TEST_DIRS', + 'TOOL_DIRS', # XXX config/Makefile.in specifies this in a make invocation #'USE_EXTENSION_MANIFEST', - b'XPCSHELL_TESTS', - b'XPIDL_MODULE', + 'XPCSHELL_TESTS', + 'XPIDL_MODULE', ] DEPRECATED_VARIABLES = [ - b'EXPORT_LIBRARY', - b'EXTRA_LIBS', - b'HOST_LIBS', - b'LIBXUL_LIBRARY', - b'MOCHITEST_A11Y_FILES', - b'MOCHITEST_BROWSER_FILES', - b'MOCHITEST_BROWSER_FILES_PARTS', - b'MOCHITEST_CHROME_FILES', - b'MOCHITEST_FILES', - b'MOCHITEST_FILES_PARTS', - b'MOCHITEST_METRO_FILES', - b'MOCHITEST_ROBOCOP_FILES', - b'MODULE_OPTIMIZE_FLAGS', - b'MOZ_CHROME_FILE_FORMAT', - b'SHORT_LIBNAME', - b'TESTING_JS_MODULES', - b'TESTING_JS_MODULE_DIR', + 'EXPORT_LIBRARY', + 'EXTRA_LIBS', + 'HOST_LIBS', + 'LIBXUL_LIBRARY', + 'MOCHITEST_A11Y_FILES', + 'MOCHITEST_BROWSER_FILES', + 'MOCHITEST_BROWSER_FILES_PARTS', + 'MOCHITEST_CHROME_FILES', + 'MOCHITEST_FILES', + 'MOCHITEST_FILES_PARTS', + 'MOCHITEST_METRO_FILES', + 'MOCHITEST_ROBOCOP_FILES', + 'MODULE_OPTIMIZE_FLAGS', + 'MOZ_CHROME_FILE_FORMAT', + 'SHORT_LIBNAME', + 'TESTING_JS_MODULES', + 'TESTING_JS_MODULE_DIR', ] MOZBUILD_VARIABLES_MESSAGE = 'It should only be defined in moz.build files.' @@ -207,7 +208,7 @@ class BackendMakeFile(object): self.fh.write(buf) def write_once(self, buf): - if isinstance(buf, unicode): + if isinstance(buf, str): buf = buf.encode('utf-8') if b'\n' + buf not in self.fh.getvalue(): self.write(buf) @@ -280,7 +281,7 @@ class RecursiveMakeTraversal(object): Helper function to call a filter from compute_dependencies and traverse. """ - return filter(current, self.get_subdirs(current)) + return list(filter(current, self.get_subdirs(current))) def compute_dependencies(self, filter=None): """ @@ -710,7 +711,7 @@ class RecursiveMakeBackend(CommonBackend): convenience variables, and the other dependency definitions for a hopefully proper directory traversal. """ - for tier, no_skip in self._no_skip.items(): + for tier, no_skip in list(self._no_skip.items()): self.log(logging.DEBUG, 'fill_root_mk', { 'number': len(no_skip), 'tier': tier }, 'Using {number} directories during {tier}') @@ -757,7 +758,7 @@ class RecursiveMakeBackend(CommonBackend): for tier, filter in filters: main, all_deps = \ self._traversal.compute_dependencies(filter) - for dir, deps in all_deps.items(): + for dir, deps in list(all_deps.items()): if deps is not None or (dir in self._idl_dirs \ and tier == 'export'): rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)]) @@ -770,7 +771,7 @@ class RecursiveMakeBackend(CommonBackend): rule.add_dependencies('%s/%s' % (d, tier) for d in main) all_compile_deps = reduce(lambda x,y: x|y, - self._compile_graph.values()) if self._compile_graph else set() + list(self._compile_graph.values())) if self._compile_graph else set() # Include the following as dependencies of the top recursion target for # compilation: # - nodes that are not dependended upon by anything. Typically, this @@ -783,7 +784,7 @@ class RecursiveMakeBackend(CommonBackend): # as direct dependencies of the top recursion target, to somehow # prioritize them. # 1. See bug 1262241 comment 5. - compile_roots = [t for t, deps in self._compile_graph.iteritems() + compile_roots = [t for t, deps in list(self._compile_graph.items()) if not deps or t not in all_compile_deps] rule = root_deps_mk.create_rule(['recurse_compile']) @@ -845,14 +846,14 @@ class RecursiveMakeBackend(CommonBackend): rule.add_dependencies(['$(CURDIR)/%: %']) def _check_blacklisted_variables(self, makefile_in, makefile_content): - if b'EXTERNALLY_MANAGED_MAKE_FILE' in makefile_content: + if 'EXTERNALLY_MANAGED_MAKE_FILE' in makefile_content: # Bypass the variable restrictions for externally managed makefiles. return for l in makefile_content.splitlines(): l = l.strip() # Don't check comments - if l.startswith(b'#'): + if l.startswith('#'): continue for x in chain(MOZBUILD_VARIABLES, DEPRECATED_VARIABLES): if x not in l: @@ -909,11 +910,11 @@ class RecursiveMakeBackend(CommonBackend): # Directories with a Makefile containing a tools target, or # XPI_PKGNAME or INSTALL_EXTENSION_ID can't be skipped and # must run during the 'tools' tier. - for t in (b'XPI_PKGNAME', b'INSTALL_EXTENSION_ID', - b'tools'): + for t in ('XPI_PKGNAME', 'INSTALL_EXTENSION_ID', + 'tools'): if t not in content: continue - if t == b'tools' and not re.search('(?:^|\s)tools.*::', content, re.M): + if t == 'tools' and not re.search('(?:^|\s)tools.*::', content, re.M): continue if objdir == self.environment.topobjdir: continue @@ -933,7 +934,7 @@ class RecursiveMakeBackend(CommonBackend): self._fill_root_mk() # Make the master test manifest files. - for flavor, t in self._test_manifests.items(): + for flavor, t in list(self._test_manifests.items()): install_prefix, manifests = t manifest_stem = mozpath.join(install_prefix, '%s.ini' % flavor) self._write_master_test_manifest(mozpath.join( @@ -1039,7 +1040,7 @@ class RecursiveMakeBackend(CommonBackend): for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done'): build_files.add_optional_exists(p) - for idl in manager.idls.values(): + for idl in list(manager.idls.values()): self._install_manifests['dist_idl'].add_link(idl['source'], idl['basename']) self._install_manifests['dist_include'].add_optional_exists('%s.h' @@ -1086,7 +1087,7 @@ class RecursiveMakeBackend(CommonBackend): interfaces_manifests = [] dist_dir = mozpath.join(self.environment.topobjdir, 'dist') - for manifest, entries in manager.interface_manifests.items(): + for manifest, entries in list(manager.interface_manifests.items()): interfaces_manifests.append(mozpath.join('$(DEPTH)', manifest)) for xpt in sorted(entries): registered_xpt_files.add(mozpath.join( @@ -1194,7 +1195,7 @@ class RecursiveMakeBackend(CommonBackend): # Don't allow files to be defined multiple times unless it is allowed. # We currently allow duplicates for non-test files or test files if # the manifest is listed as a duplicate. - for source, (dest, is_test) in obj.installs.items(): + for source, (dest, is_test) in list(obj.installs.items()): try: self._install_manifests['_test_files'].add_link(source, dest) except ValueError: @@ -1558,7 +1559,7 @@ class RecursiveMakeBackend(CommonBackend): man_dir = mozpath.join(self.environment.topobjdir, '_build_manifests', dest) - for k, manifest in manifests.items(): + for k, manifest in list(manifests.items()): with self._write_file(mozpath.join(man_dir, k)) as fh: manifest.write(fileobj=fh) @@ -1593,20 +1594,20 @@ class RecursiveMakeBackend(CommonBackend): pp.context.update(extra) if not pp.context.get('autoconfmk', ''): pp.context['autoconfmk'] = 'autoconf.mk' - pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n'); - pp.handleLine(b'DEPTH := @DEPTH@\n') - pp.handleLine(b'topobjdir := @topobjdir@\n') - pp.handleLine(b'topsrcdir := @top_srcdir@\n') - pp.handleLine(b'srcdir := @srcdir@\n') - pp.handleLine(b'VPATH := @srcdir@\n') - pp.handleLine(b'relativesrcdir := @relativesrcdir@\n') - pp.handleLine(b'include $(DEPTH)/config/@autoconfmk@\n') + pp.handleLine('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n'); + pp.handleLine('DEPTH := @DEPTH@\n') + pp.handleLine('topobjdir := @topobjdir@\n') + pp.handleLine('topsrcdir := @top_srcdir@\n') + pp.handleLine('srcdir := @srcdir@\n') + pp.handleLine('VPATH := @srcdir@\n') + pp.handleLine('relativesrcdir := @relativesrcdir@\n') + pp.handleLine('include $(DEPTH)/config/@autoconfmk@\n') if not stub: pp.do_include(obj.input_path) # Empty line to avoid failures when last line in Makefile.in ends # with a backslash. - pp.handleLine(b'\n') - pp.handleLine(b'include $(topsrcdir)/config/recurse.mk\n') + pp.handleLine('\n') + pp.handleLine('include $(topsrcdir)/config/recurse.mk\n') if not stub: # Adding the Makefile.in here has the desired side-effect # that if the Makefile.in disappears, this will force diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py index d46f1332d..a9a27a699 100644 --- a/python/mozbuild/mozbuild/config_status.py +++ b/python/mozbuild/mozbuild/config_status.py @@ -77,6 +77,7 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, See build/autoconf/config.status.m4. ''' + print("config_status started") if 'CONFIG_FILES' in os.environ: raise Exception('Using the CONFIG_FILES environment variable is not ' 'supported.') @@ -119,7 +120,7 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, if 'WRITE_MOZINFO' in os.environ: write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ) - cpu_start = time.clock() + cpu_start = time.perf_counter() time_start = time.time() # Make appropriate backend instances, defaulting to RecursiveMakeBackend, @@ -155,7 +156,7 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, summary = obj.gyp_summary() print(summary, file=sys.stderr) - cpu_time = time.clock() - cpu_start + cpu_time = time.perf_counter() - cpu_start wall_time = time.time() - time_start efficiency = cpu_time / wall_time if wall_time else 100 untracked = wall_time - execution_time @@ -179,3 +180,5 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, # Advertise Android Studio if it is appropriate. if MachCommandConditions.is_android(env): print(ANDROID_IDE_ADVERTISEMENT) + + print("config_status finished") diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py index d03615707..13d623d4f 100644 --- a/python/mozbuild/mozbuild/configure/__init__.py +++ b/python/mozbuild/mozbuild/configure/__init__.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals -import __builtin__ + +import builtins import inspect import logging import os @@ -38,6 +38,8 @@ from mozbuild.util import ( import mozpack.path as mozpath +import traceback + class ConfigureError(Exception): pass @@ -69,7 +71,7 @@ class SandboxDependsFunction(object): def __getattr__(self, key): return self._getattr(key).sandboxed - def __nonzero__(self): + def __bool__(self): raise ConfigureError( 'Cannot do boolean operations on @depends functions.') @@ -96,6 +98,7 @@ class DependsFunction(object): sandbox._value_for(self) elif not sandbox._help: sandbox._execution_queue.append((sandbox._value_for, (self,))) + sandbox.tasks_debug_out("DependsFunction.__init %s" % func.__name__) @property def name(self): @@ -206,6 +209,15 @@ class CombinedDependsFunction(DependsFunction): def __ne__(self, other): return not self == other + def __hash__(self): + # This was one was taken from [1] initially. Should not have done that: + # it causes explosion of ConfigureSandbox._execution_queue with 100% + # CPU load and eating all avaliable memory... + # + # [1] https://code.foxkit.us/adelie/packages/blob/f2b5773da19ab397fbe64fd32dacc383cfe4cd77/user/mozjs/python3.patch#L8068 + return hash((self._name, tuple(self.dependencies))) + + class SandboxedGlobal(dict): '''Identifiable dict type for use as function global''' @@ -253,11 +265,12 @@ class ConfigureSandbox(dict): # The default set of builtins. We expose unicode as str to make sandboxed # files more python3-ready. BUILTINS = ReadOnlyDict({ - b: getattr(__builtin__, b) + b: getattr(builtins, b) for b in ('None', 'False', 'True', 'int', 'bool', 'any', 'all', 'len', 'list', 'tuple', 'set', 'dict', 'isinstance', 'getattr', - 'hasattr', 'enumerate', 'range', 'zip') - }, __import__=forbidden_import, str=unicode) + 'hasattr', 'enumerate', 'range', 'zip', '__build_class__', + 'bytes', 'exec') + }, __import__=forbidden_import, str=str) # Expose a limited set of functions from os.path OS = ReadOnlyNamespace(path=ReadOnlyNamespace(**{ @@ -294,6 +307,11 @@ class ConfigureSandbox(dict): # Queue of functions to execute, with their arguments self._execution_queue = [] + # For debugging: Show number of tasks started in run() / added elsewhere + # and some additional info + self.task_debug = False # set True to enable + self.tasks_started = 0 + # Store the `when`s associated to some options. self._conditions = {} @@ -331,7 +349,7 @@ class ConfigureSandbox(dict): return method def wrapped(*args, **kwargs): out_args = [ - arg.decode(encoding) if isinstance(arg, str) else arg + arg.decode(encoding) if isinstance(arg, bytes) else arg for arg in args ] return method(*out_args, **kwargs) @@ -360,6 +378,14 @@ class ConfigureSandbox(dict): handler.setFormatter(formatter) logger.addHandler(handler) + def tasks_debug_out(self, text): + if self.task_debug: + print("%s / queued %i / done %i" %(text, len(self._execution_queue), self.tasks_started)) + #if len(self._execution_queue) > 5000: + # traceback.print_stack(file=sys.stdout) + #if len(self._execution_queue) > 5010: + # raise Exception("Too many tasks") + def include_file(self, path): '''Include one file in the sandbox. Users of this class probably want to use `run` instead. @@ -380,6 +406,9 @@ class ConfigureSandbox(dict): if path in self._all_paths: raise ConfigureError( 'Cannot include `%s` because it was included already.' % path) + + if self.task_debug: + print("include_file", path) self._paths.append(path) self._all_paths.add(path) @@ -398,7 +427,7 @@ class ConfigureSandbox(dict): if path: self.include_file(path) - for option in self._options.itervalues(): + for option in self._options.values(): # All options must be referenced by some @depends function if option not in self._seen: raise ConfigureError( @@ -425,6 +454,8 @@ class ConfigureSandbox(dict): # Run the execution queue for func, args in self._execution_queue: + self.tasks_started += 1 + self.tasks_debug_out("ConfigureSandbox.run(%s)" % func.__name__) func(*args) if self._help: @@ -504,7 +535,7 @@ class ConfigureSandbox(dict): value = PositiveOptionValue() elif value is False or value == (): value = NegativeOptionValue() - elif isinstance(value, types.StringTypes): + elif isinstance(value, (str,)): value = PositiveOptionValue((value,)) elif isinstance(value, tuple): value = PositiveOptionValue(value) @@ -544,7 +575,7 @@ class ConfigureSandbox(dict): return value def _dependency(self, arg, callee_name, arg_name=None): - if isinstance(arg, types.StringTypes): + if isinstance(arg, (str,)): prefix, name, values = Option.split_option(arg) if values != (): raise ConfigureError("Option must not contain an '='") @@ -608,7 +639,7 @@ class ConfigureSandbox(dict): ''' when = self._normalize_when(kwargs.get('when'), 'option') args = [self._resolve(arg) for arg in args] - kwargs = {k: self._resolve(v) for k, v in kwargs.iteritems() + kwargs = {k: self._resolve(v) for k, v in kwargs.items() if k != 'when'} option = Option(*args, **kwargs) if when: @@ -689,7 +720,7 @@ class ConfigureSandbox(dict): with self.only_when_impl(when): what = self._resolve(what) if what: - if not isinstance(what, types.StringTypes): + if not isinstance(what, (str,)): raise TypeError("Unexpected type: '%s'" % type(what).__name__) self.include_file(what) @@ -707,7 +738,7 @@ class ConfigureSandbox(dict): (k[:-len('_impl')], getattr(self, k)) for k in dir(self) if k.endswith('_impl') and k != 'template_impl' ) - glob.update((k, v) for k, v in self.iteritems() if k not in glob) + glob.update((k, v) for k, v in self.items() if k not in glob) # Any function argument to the template must be prepared to be sandboxed. # If the template itself returns a function (in which case, it's very @@ -731,7 +762,7 @@ class ConfigureSandbox(dict): def wrapper(*args, **kwargs): args = [maybe_prepare_function(arg) for arg in args] kwargs = {k: maybe_prepare_function(v) - for k, v in kwargs.iteritems()} + for k, v in kwargs.items()} ret = template(*args, **kwargs) if isfunction(ret): # We can't expect the sandboxed code to think about all the @@ -766,7 +797,7 @@ class ConfigureSandbox(dict): for value, required in ( (_import, True), (_from, False), (_as, False)): - if not isinstance(value, types.StringTypes) and ( + if not isinstance(value, (str,)) and ( required or value is not None): raise TypeError("Unexpected type: '%s'" % type(value).__name__) if value is not None and not self.RE_MODULE.match(value): @@ -807,7 +838,7 @@ class ConfigureSandbox(dict): # Special case for the open() builtin, because otherwise, using it # fails with "IOError: file() constructor not accessible in # restricted mode" - if what == '__builtin__.open': + if what == 'builtins.open': return lambda *args, **kwargs: open(*args, **kwargs) # Until this proves to be a performance problem, just construct an # import statement and execute it. @@ -829,7 +860,7 @@ class ConfigureSandbox(dict): name = self._resolve(name, need_help_dependency=False) if name is None: return - if not isinstance(name, types.StringTypes): + if not isinstance(name, (str,)): raise TypeError("Unexpected type: '%s'" % type(name).__name__) if name in data: raise ConfigureError( @@ -850,6 +881,7 @@ class ConfigureSandbox(dict): self._execution_queue.append(( self._resolve_and_set, (self._config, name, value, when))) + self.tasks_debug_out("ConfigureSandbox.set_config_impl / %s(%s" % (name, value)) def set_define_impl(self, name, value, when=None): '''Implementation of set_define(). @@ -864,6 +896,7 @@ class ConfigureSandbox(dict): defines = self._config.setdefault('DEFINES', {}) self._execution_queue.append(( self._resolve_and_set, (defines, name, value, when))) + self.tasks_debug_out("ConfigureSandbox.set_define_impl / %s(%s)" % (name, value)) def imply_option_impl(self, option, value, reason=None, when=None): '''Implementation of imply_option(). @@ -922,7 +955,7 @@ class ConfigureSandbox(dict): if isinstance(possible_reasons[0], Option): reason = possible_reasons[0] if not reason and (isinstance(value, (bool, tuple)) or - isinstance(value, types.StringTypes)): + isinstance(value, (str,))): # A reason can be provided automatically when imply_option # is called with an immediate value. _, filename, line, _, _, _ = inspect.stack()[1] @@ -955,10 +988,10 @@ class ConfigureSandbox(dict): if not inspect.isfunction(func): raise TypeError("Unexpected type: '%s'" % type(func).__name__) if func in self._prepared_functions: - return func, func.func_globals + return func, func.__globals__ glob = SandboxedGlobal( - (k, v) for k, v in func.func_globals.iteritems() + (k, v) for k, v in func.__globals__.items() if (inspect.isfunction(v) and v not in self._templates) or ( inspect.isclass(v) and issubclass(v, Exception)) ) @@ -979,20 +1012,20 @@ class ConfigureSandbox(dict): # Note this is not entirely bullet proof (if the value is e.g. a list, # the list contents could have changed), but covers the bases. closure = None - if func.func_closure: + if func.__closure__: def makecell(content): def f(): content - return f.func_closure[0] + return f.__closure__[0] closure = tuple(makecell(cell.cell_contents) - for cell in func.func_closure) + for cell in func.__closure__) new_func = self.wraps(func)(types.FunctionType( - func.func_code, + func.__code__, glob, func.__name__, - func.func_defaults, + func.__defaults__, closure )) @self.wraps(new_func) diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py index c0caa9cc5..a3e1f37e1 100644 --- a/python/mozbuild/mozbuild/configure/check_debug_ranges.py +++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py @@ -6,7 +6,7 @@ # to a given compilation unit. This is used as a helper to find a bug in some # versions of GNU ld. -from __future__ import absolute_import + import subprocess import sys @@ -45,6 +45,8 @@ def get_range_length(range, debug_ranges): def main(bin, compilation_unit): p = subprocess.Popen(['objdump', '-W', bin], stdout = subprocess.PIPE, stderr = subprocess.PIPE) (out, err) = p.communicate() + if isinstance(out, bytes): + out = out.decode('utf-8') sections = re.split('\n(Contents of the|The section) ', out) debug_info = [s for s in sections if s.startswith('.debug_info')] debug_ranges = [s for s in sections if s.startswith('.debug_ranges')] @@ -59,4 +61,4 @@ def main(bin, compilation_unit): if __name__ == '__main__': - print(main(*sys.argv[1:])) + print((main(*sys.argv[1:]))) diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py index 53ae2ae6d..4d80cad86 100644 --- a/python/mozbuild/mozbuild/configure/options.py +++ b/python/mozbuild/mozbuild/configure/options.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import os import sys @@ -12,7 +12,7 @@ from collections import OrderedDict def istupleofstrings(obj): return isinstance(obj, tuple) and len(obj) and all( - isinstance(o, types.StringTypes) for o in obj) + isinstance(o, (str,)) for o in obj) class OptionValue(tuple): @@ -88,7 +88,7 @@ class PositiveOptionValue(OptionValue): in the form of a tuple for when values are given to the option (in the form --option=value[,value2...]. ''' - def __nonzero__(self): + def __bool__(self): return True @@ -113,7 +113,7 @@ class ConflictingOptionError(InvalidOptionError): if format_data: message = message.format(**format_data) super(ConflictingOptionError, self).__init__(message) - for k, v in format_data.iteritems(): + for k, v in format_data.items(): setattr(self, k, v) @@ -149,7 +149,7 @@ class Option(object): 'At least an option name or an environment variable name must ' 'be given') if name: - if not isinstance(name, types.StringTypes): + if not isinstance(name, (str,)): raise InvalidOptionError('Option must be a string') if not name.startswith('--'): raise InvalidOptionError('Option must start with `--`') @@ -158,7 +158,7 @@ class Option(object): if not name.islower(): raise InvalidOptionError('Option must be all lowercase') if env: - if not isinstance(env, types.StringTypes): + if not isinstance(env, (str,)): raise InvalidOptionError( 'Environment variable name must be a string') if not env.isupper(): @@ -168,8 +168,8 @@ class Option(object): isinstance(nargs, int) and nargs >= 0): raise InvalidOptionError( "nargs must be a positive integer, '?', '*' or '+'") - if (not isinstance(default, types.StringTypes) and - not isinstance(default, (bool, types.NoneType)) and + if (not isinstance(default, (str,)) and + not isinstance(default, (bool, type(None))) and not istupleofstrings(default)): raise InvalidOptionError( 'default must be a bool, a string or a tuple of strings') @@ -241,7 +241,7 @@ class Option(object): ', '.join("'%s'" % c for c in choices)) elif has_choices: maxargs = self.maxargs - if len(choices) < maxargs and maxargs != sys.maxint: + if len(choices) < maxargs and maxargs != sys.maxsize: raise InvalidOptionError('Not enough `choices` for `nargs`') self.choices = choices self.help = help @@ -255,7 +255,7 @@ class Option(object): where prefix is one of 'with', 'without', 'enable' or 'disable'. The '=values' part is optional. Values are separated with commas. ''' - if not isinstance(option, types.StringTypes): + if not isinstance(option, (str,)): raise InvalidOptionError('Option must be a string') elements = option.split('=', 1) @@ -308,7 +308,7 @@ class Option(object): def maxargs(self): if isinstance(self.nargs, int): return self.nargs - return 1 if self.nargs == '?' else sys.maxint + return 1 if self.nargs == '?' else sys.maxsize def _validate_nargs(self, num): minargs, maxargs = self.minargs, self.maxargs @@ -499,5 +499,5 @@ class CommandLineHelper(object): def __iter__(self): for d in (self._args, self._extra_args): - for arg, pos in d.itervalues(): + for arg, pos in d.values(): yield arg diff --git a/python/mozbuild/mozbuild/configure/util.py b/python/mozbuild/mozbuild/configure/util.py index 9d8b2eb0e..a12986e48 100644 --- a/python/mozbuild/mozbuild/configure/util.py +++ b/python/mozbuild/mozbuild/configure/util.py @@ -77,15 +77,7 @@ class ConfigureOutputHandler(logging.Handler): # Python has this feature where it sets the encoding of pipes to # ascii, which blatantly fails when trying to print out non-ascii. def fix_encoding(fh): - try: - isatty = fh.isatty() - except AttributeError: - isatty = True - - if not isatty: - encoding = getpreferredencoding() - if encoding: - return codecs.getwriter(encoding)(fh) + # no magic on oe / python3 return fh self._stdout = fix_encoding(stdout) @@ -200,7 +192,7 @@ class LineIO(object): self._errors = errors def write(self, buf): - if self._encoding and isinstance(buf, str): + if self._encoding and isinstance(buf, bytes): buf = buf.decode(self._encoding, self._errors) lines = buf.splitlines() if not lines: diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index d5af532f7..e9810fe58 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals + import errno import getpass @@ -146,7 +146,7 @@ class TierStatus(object): """ o = [] - for tier, state in self.tiers.items(): + for tier, state in list(self.tiers.items()): t_entry = dict( name=tier, start=state['begin_time'], @@ -574,7 +574,7 @@ class BuildProgressFooter(Footer): def __init__(self, terminal, monitor): Footer.__init__(self, terminal) - self.tiers = monitor.tiers.tier_status.viewitems() + self.tiers = monitor.tiers.tier_status.items() def draw(self): """Draws this footer in the terminal.""" @@ -911,8 +911,8 @@ class CCacheStats(object): return '\n'.join(lines) - def __nonzero__(self): - relative_values = [v for k, v in self._values.items() + def __bool__(self): + relative_values = [v for k, v in list(self._values.items()) if k not in self.ABSOLUTE_KEYS] return (all(v >= 0 for v in relative_values) and any(v > 0 for v in relative_values)) @@ -1156,7 +1156,7 @@ class BuildDriver(MozbuildObject): high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: - print(FINDER_SLOW_MESSAGE % finder_percent) + print((FINDER_SLOW_MESSAGE % finder_percent)) ccache_end = monitor.ccache_stats() @@ -1276,8 +1276,8 @@ class BuildDriver(MozbuildObject): """Install test files.""" if self.is_clobber_needed(): - print(INSTALL_TESTS_CLOBBER.format( - clobber_file=os.path.join(self.topobjdir, 'CLOBBER'))) + print((INSTALL_TESTS_CLOBBER.format( + clobber_file=os.path.join(self.topobjdir, 'CLOBBER')))) sys.exit(1) if not test_objs: diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index fbdbefc1d..1aef6a65a 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -14,7 +14,7 @@ If you are looking for the absolute authority on what moz.build files can contain, you've come to the right place. """ -from __future__ import absolute_import, unicode_literals + import os @@ -237,15 +237,15 @@ class Context(KeyedDefaultDict): This function is transactional: if setitem fails for one of the values, the context is not updated at all.""" if isinstance(iterable, dict): - iterable = iterable.items() + iterable = list(iterable.items()) update = {} - for key, value in itertools.chain(iterable, kwargs.items()): + for key, value in itertools.chain(iterable, list(kwargs.items())): stored_type = self._validate(key, value) # Don't create an instance of stored_type if coercion is needed, # until all values are validated. update[key] = (value, stored_type) - for key, (value, stored_type) in update.items(): + for key, (value, stored_type) in list(update.items()): if not isinstance(value, stored_type): update[key] = stored_type(value) else: @@ -311,7 +311,7 @@ class BaseCompileFlags(ContextDerivedValue, dict): # a template were set and which were provided as defaults. template_name = getattr(context, 'template', None) if template_name in (None, 'Gyp'): - dict.__init__(self, ((k, v if v is None else TypedList(unicode)(v)) + dict.__init__(self, ((k, v if v is None else TypedList(str)(v)) for k, v, _ in self.flag_variables)) else: dict.__init__(self) @@ -520,13 +520,13 @@ class CompileFlags(BaseCompileFlags): if key in self and self[key] is None: raise ValueError('`%s` may not be set in COMPILE_FLAGS from moz.build, this ' 'value is resolved from the emitter.' % key) - if not (isinstance(value, list) and all(isinstance(v, basestring) for v in value)): + if not (isinstance(value, list) and all(isinstance(v, str) for v in value)): raise ValueError('A list of strings must be provided as a value for a ' 'compile flags category.') dict.__setitem__(self, key, value) -class FinalTargetValue(ContextDerivedValue, unicode): +class FinalTargetValue(ContextDerivedValue, str): def __new__(cls, context, value=""): if not value: value = 'dist/' @@ -536,7 +536,7 @@ class FinalTargetValue(ContextDerivedValue, unicode): value += 'bin' if context['DIST_SUBDIR']: value += '/' + context['DIST_SUBDIR'] - return unicode.__new__(cls, value) + return str.__new__(cls, value) def Enum(*values): @@ -584,7 +584,7 @@ class PathMeta(type): cls = SourcePath return super(PathMeta, cls).__call__(context, value) -class Path(ContextDerivedValue, unicode): +class Path(ContextDerivedValue, str, metaclass=PathMeta): """Stores and resolves a source path relative to a given context This class is used as a backing type for some of the sandbox variables. @@ -595,7 +595,6 @@ class Path(ContextDerivedValue, unicode): - '!objdir/relative/paths' - '%/filesystem/absolute/paths' """ - __metaclass__ = PathMeta def __new__(cls, context, value=None): return super(Path, cls).__new__(cls, value) @@ -612,10 +611,14 @@ class Path(ContextDerivedValue, unicode): """ return Path(self.context, mozpath.join(self, *p)) + @staticmethod + def cmp(a, b): + return (a > b) - (a < b) + def __cmp__(self, other): if isinstance(other, Path) and self.srcdir != other.srcdir: - return cmp(self.full_path, other.full_path) - return cmp(unicode(self), other) + return self.cmp(self.full_path, other.full_path) + return self.cmp(str(self), other) # __cmp__ is not enough because unicode has __eq__, __ne__, etc. defined # and __cmp__ is only used for those when they don't exist. @@ -773,7 +776,7 @@ def ContextDerivedTypedRecord(*fields): __slots__ = tuple([name for name, _ in fields]) def __init__(self, context): - for fname, ftype in self._fields.items(): + for fname, ftype in list(self._fields.items()): if issubclass(ftype, ContextDerivedValue): setattr(self, fname, self._fields[fname](context)) else: @@ -909,8 +912,8 @@ def TypedListWithAction(typ, action): return _TypedListWithAction WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest", - [("manifest_path", unicode), - ("test_root", unicode)]) + [("manifest_path", str), + ("test_root", str)]) ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest) ReftestManifestList = OrderedPathListWithAction(read_reftest_manifest) WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest) @@ -918,18 +921,18 @@ WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList) OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()), StrictOrderingOnAppendList) -OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList) +OrderedStringList = TypedList(str, StrictOrderingOnAppendList) DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList), ('tags', OrderedStringList), ('flavors', OrderedTestFlavorList)) BugzillaComponent = TypedNamedTuple('BugzillaComponent', - [('product', unicode), ('component', unicode)]) + [('product', str), ('component', str)]) SchedulingComponents = ContextDerivedTypedRecord( - ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)), - ('exclusive', TypedList(unicode, StrictOrderingOnAppendList))) + ('inclusive', TypedList(str, StrictOrderingOnAppendList)), + ('exclusive', TypedList(str, StrictOrderingOnAppendList))) GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({ - 'script': unicode, + 'script': str, 'inputs': list, 'flags': list, }) @@ -1096,7 +1099,7 @@ class Files(SubContext): self.test_tags |= other.test_tags self.test_flavors |= other.test_flavors - for k, v in other.items(): + for k, v in list(other.items()): if k == 'IMPACTED_TESTS': self.test_files |= set(mozpath.relpath(e.full_path, e.context.config.topsrcdir) for e in v.files) @@ -1154,7 +1157,7 @@ class Files(SubContext): bug_components = Counter() - for f in files.values(): + for f in list(files.values()): bug_component = f.get('BUG_COMPONENT') if bug_component: bug_components[bug_component] += 1 @@ -1232,7 +1235,7 @@ VARIABLES = { RustLibrary template instead. """), - 'RUST_LIBRARY_TARGET_DIR': (unicode, unicode, + 'RUST_LIBRARY_TARGET_DIR': (str, str, """Where CARGO_TARGET_DIR should point when compiling this library. If not set, it defaults to the current objdir. It should be a relative path to the current objdir; absolute paths should not be used. @@ -1248,7 +1251,7 @@ VARIABLES = { HostRustLibrary template instead. """), - 'RUST_TEST': (unicode, unicode, + 'RUST_TEST': (str, str, """Name of a Rust test to build and run via `cargo test`. This variable should not be used directly; you should be using the @@ -1487,7 +1490,7 @@ VARIABLES = { """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. """), - 'FINAL_LIBRARY': (unicode, unicode, + 'FINAL_LIBRARY': (str, str, """Library in which the objects of the current directory will be linked. This variable contains the name of a library, defined elsewhere with @@ -1528,7 +1531,7 @@ VARIABLES = { with the host compiler. """), - 'HOST_LIBRARY_NAME': (unicode, unicode, + 'HOST_LIBRARY_NAME': (str, str, """Name of target library generated when cross compiling. """), @@ -1546,7 +1549,7 @@ VARIABLES = { libraries that link into this library via FINAL_LIBRARY. """), - 'LIBRARY_NAME': (unicode, unicode, + 'LIBRARY_NAME': (str, str, """The code name of the library generated for a directory. By default STATIC_LIBRARY_NAME and SHARED_LIBRARY_NAME take this name. @@ -1558,7 +1561,7 @@ VARIABLES = { ``example/components/xpcomsample.lib`` on Windows. """), - 'SHARED_LIBRARY_NAME': (unicode, unicode, + 'SHARED_LIBRARY_NAME': (str, str, """The name of the static library generated for a directory, if it needs to differ from the library code name. @@ -1572,7 +1575,7 @@ VARIABLES = { Implies FORCE_SHARED_LIB. """), - 'STATIC_LIBRARY_NAME': (unicode, unicode, + 'STATIC_LIBRARY_NAME': (str, str, """The name of the static library generated for a directory, if it needs to differ from the library code name. @@ -1604,31 +1607,31 @@ VARIABLES = { This variable contains a list of system libaries to link against. """), - 'RCFILE': (unicode, unicode, + 'RCFILE': (str, str, """The program .rc file. This variable can only be used on Windows. """), - 'RESFILE': (unicode, unicode, + 'RESFILE': (str, str, """The program .res file. This variable can only be used on Windows. """), - 'RCINCLUDE': (unicode, unicode, + 'RCINCLUDE': (str, str, """The resource script file to be included in the default .res file. This variable can only be used on Windows. """), - 'DEFFILE': (unicode, unicode, + 'DEFFILE': (str, str, """The program .def (module definition) file. This variable can only be used on Windows. """), - 'SYMBOLS_FILE': (Path, unicode, + 'SYMBOLS_FILE': (Path, str, """A file containing a list of symbols to export from a shared library. The given file contains a list of symbols to be exported, and is @@ -1649,7 +1652,7 @@ VARIABLES = { ``BIN_SUFFIX``, the name will remain unchanged. """), - 'SONAME': (unicode, unicode, + 'SONAME': (str, str, """The soname of the shared object currently being linked soname is the "logical name" of a shared object, often used to provide @@ -1719,7 +1722,7 @@ VARIABLES = { ``GENERATED_FILES``. """), - 'PROGRAM' : (unicode, unicode, + 'PROGRAM' : (str, str, """Compiled executable name. If the configuration token ``BIN_SUFFIX`` is set, its value will be @@ -1727,7 +1730,7 @@ VARIABLES = { ``BIN_SUFFIX``, ``PROGRAM`` will remain unchanged. """), - 'HOST_PROGRAM' : (unicode, unicode, + 'HOST_PROGRAM' : (str, str, """Compiled host executable name. If the configuration token ``HOST_BIN_SUFFIX`` is set, its value will be @@ -1765,7 +1768,7 @@ VARIABLES = { files. """), - 'XPIDL_MODULE': (unicode, unicode, + 'XPIDL_MODULE': (str, str, """XPCOM Interface Definition Module Name. This is the name of the ``.xpt`` file that is created by linking @@ -1924,14 +1927,14 @@ VARIABLES = { # The following variables are used to control the target of installed files. - 'XPI_NAME': (unicode, unicode, + 'XPI_NAME': (str, str, """The name of an extension XPI to generate. When this variable is present, the results of this directory will end up being packaged into an extension instead of the main dist/bin results. """), - 'DIST_SUBDIR': (unicode, unicode, + 'DIST_SUBDIR': (str, str, """The name of an alternate directory to install files to. When this variable is present, the results of this directory will end up @@ -1939,7 +1942,7 @@ VARIABLES = { otherwise be placed. """), - 'FINAL_TARGET': (FinalTargetValue, unicode, + 'FINAL_TARGET': (FinalTargetValue, str, """The name of the directory to install targets to. The directory is relative to the top of the object directory. The @@ -1970,7 +1973,7 @@ VARIABLES = { 'GYP_DIRS': (StrictOrderingOnAppendListWithFlagsFactory({ 'variables': dict, - 'input': unicode, + 'input': str, 'sandbox_vars': dict, 'no_chromium': bool, 'no_unified': bool, @@ -2194,7 +2197,7 @@ VARIABLES = { } # Sanity check: we don't want any variable above to have a list as storage type. -for name, (storage_type, input_types, docs) in VARIABLES.items(): +for name, (storage_type, input_types, docs) in list(VARIABLES.items()): if storage_type == list: raise RuntimeError('%s has a "list" storage type. Use "List" instead.' % name) diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 442fc9e0a..837453a9f 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -15,7 +15,7 @@ contains the code for converting executed mozbuild files into these data structures. """ -from __future__ import absolute_import, unicode_literals + from mozbuild.util import StrictOrderingOnAppendList from mozpack.chrome.manifest import ManifestEntry @@ -182,7 +182,7 @@ class ComputedFlags(ContextDerived): if value: for dest_var in dest_vars: flags[dest_var].extend(value) - return flags.items() + return list(flags.items()) class XPIDLFile(ContextDerived): """Describes an XPIDL file to be compiled.""" @@ -213,7 +213,7 @@ class BaseDefines(ContextDerived): self.defines = defines def get_defines(self): - for define, value in self.defines.iteritems(): + for define, value in self.defines.items(): if value is True: yield('-D%s' % define) elif value is False: @@ -494,7 +494,7 @@ class SimpleProgram(BaseProgram): KIND = 'target' def source_files(self): - for srcs in self.sources.values(): + for srcs in list(self.sources.values()): for f in srcs: if mozpath.basename(mozpath.splitext(f)[0]) == mozpath.splitext(self.program)[0]: return [f] diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 642b381c0..c28344a75 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals + import itertools import logging @@ -116,8 +116,8 @@ class TreeMetadataEmitter(LoggingMixin): # arguments. This gross hack works around the problem until we # rid ourselves of 2.6. self.info = {} - for k, v in mozinfo.info.items(): - if isinstance(k, unicode): + for k, v in list(mozinfo.info.items()): + if isinstance(k, str): k = k.encode('ascii') self.info[k] = v @@ -234,7 +234,7 @@ class TreeMetadataEmitter(LoggingMixin): # Next do FINAL_LIBRARY linkage. - for lib in (l for libs in self._libs.values() for l in libs): + for lib in (l for libs in list(self._libs.values()) for l in libs): if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: continue if lib.link_into not in self._libs: @@ -294,13 +294,13 @@ class TreeMetadataEmitter(LoggingMixin): lib.link_into == outerlib.basename): propagate_defines(lib, defines) - for lib in (l for libs in self._libs.values() for l in libs): + for lib in (l for libs in list(self._libs.values()) for l in libs): if isinstance(lib, Library): propagate_defines(lib, lib.lib_defines) yield lib - for lib in (l for libs in self._libs.values() for l in libs): + for lib in (l for libs in list(self._libs.values()) for l in libs): lib_defines = list(lib.lib_defines.get_defines()) if lib_defines: objdir_flags = self._compile_flags[lib.objdir] @@ -310,13 +310,13 @@ class TreeMetadataEmitter(LoggingMixin): if objdir_flags: objdir_flags.resolve_flags('LIBRARY_DEFINES', lib_defines) - for flags_obj in self._compile_flags.values(): + for flags_obj in list(self._compile_flags.values()): yield flags_obj - for flags_obj in self._compile_as_flags.values(): + for flags_obj in list(self._compile_as_flags.values()): yield flags_obj - for obj in self._binaries.values(): + for obj in list(self._binaries.values()): yield obj @@ -409,7 +409,7 @@ class TreeMetadataEmitter(LoggingMixin): libs[key] = l if key not in libs: libs[key] = l - candidates = libs.values() + candidates = list(libs.values()) if force_static and not candidates: if dir: raise SandboxValidationError( @@ -473,9 +473,9 @@ class TreeMetadataEmitter(LoggingMixin): def _verify_deps(self, context, crate_dir, crate_name, dependencies, description='Dependency'): """Verify that a crate's dependencies all specify local paths.""" - for dep_crate_name, values in dependencies.iteritems(): + for dep_crate_name, values in dependencies.items(): # A simple version number. - if isinstance(values, (str, unicode)): + if isinstance(values, str): raise SandboxValidationError( '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), context) @@ -529,7 +529,7 @@ class TreeMetadataEmitter(LoggingMixin): cargo_target_dir = context.get('RUST_LIBRARY_TARGET_DIR', '.') - dependencies = set(config.get('dependencies', {}).iterkeys()) + dependencies = set(config.get('dependencies', {}).keys()) features = context.get(cls.FEATURES_VAR, []) unique_features = set(features) @@ -863,7 +863,7 @@ class TreeMetadataEmitter(LoggingMixin): assert not gen_sources['UNIFIED_SOURCES'] no_pgo = context.get('NO_PGO') - no_pgo_sources = [f for f, flags in all_flags.iteritems() + no_pgo_sources = [f for f, flags in all_flags.items() if flags.no_pgo] if no_pgo: if no_pgo_sources: @@ -890,7 +890,7 @@ class TreeMetadataEmitter(LoggingMixin): # The inverse of the above, mapping suffixes to their canonical suffix. canonicalized_suffix_map = {} - for suffix, alternatives in suffix_map.iteritems(): + for suffix, alternatives in suffix_map.items(): alternatives.add(suffix) for a in alternatives: canonicalized_suffix_map[a] = suffix @@ -914,7 +914,7 @@ class TreeMetadataEmitter(LoggingMixin): # Source files to track for linkables associated with this context. ctxt_sources = defaultdict(lambda: defaultdict(list)) - for variable, (klass, gen_klass, suffixes) in varmap.items(): + for variable, (klass, gen_klass, suffixes) in list(varmap.items()): allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes]) # First ensure that we haven't been given filetypes that we don't @@ -941,20 +941,20 @@ class TreeMetadataEmitter(LoggingMixin): obj = cls(*arglist) srcs = obj.files if isinstance(obj, UnifiedSources) and obj.have_unified_mapping: - srcs = dict(obj.unified_source_mapping).keys() + srcs = list(dict(obj.unified_source_mapping).keys()) ctxt_sources[variable][canonical_suffix] += sorted(srcs) yield obj if ctxt_sources: for linkable in linkables: for target_var in ('SOURCES', 'UNIFIED_SOURCES'): - for suffix, srcs in ctxt_sources[target_var].items(): + for suffix, srcs in list(ctxt_sources[target_var].items()): linkable.sources[suffix] += srcs for host_linkable in host_linkables: - for suffix, srcs in ctxt_sources['HOST_SOURCES'].items(): + for suffix, srcs in list(ctxt_sources['HOST_SOURCES'].items()): host_linkable.sources[suffix] += srcs - for f, flags in all_flags.iteritems(): + for f, flags in all_flags.items(): if flags.flags: ext = mozpath.splitext(f)[1] yield PerSourceFlag(context, f, flags.flags) @@ -1143,7 +1143,7 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._handle_linkables(context, passthru, generated_files): yield obj - generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) + generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in list(self._binaries.keys())]) components = [] for var, cls in ( @@ -1277,7 +1277,7 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._process_jar_manifests(context): yield obj - for name, jar in context.get('JAVA_JAR_TARGETS', {}).items(): + for name, jar in list(context.get('JAVA_JAR_TARGETS', {}).items()): yield ContextWrapped(context, jar) computed_as_flags.resolve_flags('MOZBUILD', @@ -1346,7 +1346,7 @@ class TreeMetadataEmitter(LoggingMixin): script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)), 'action', 'process_define_files.py') yield GeneratedFile(context, script, 'process_define_file', - unicode(path), + str(path), [Path(context, path + '.in')]) generated_files = context.get('GENERATED_FILES') or [] @@ -1393,7 +1393,7 @@ class TreeMetadataEmitter(LoggingMixin): flags.flags, localized=localized) def _process_test_manifests(self, context): - for prefix, info in TEST_MANIFESTS.items(): + for prefix, info in list(TEST_MANIFESTS.items()): for path, manifest in context.get('%s_MANIFESTS' % prefix, []): for obj in self._process_test_manifest(context, info, path, manifest): yield obj @@ -1479,7 +1479,7 @@ class TreeMetadataEmitter(LoggingMixin): process_support_files(test) - for path, m_defaults in mpmanifest.manifest_defaults.items(): + for path, m_defaults in list(mpmanifest.manifest_defaults.items()): process_support_files(m_defaults) # We also copy manifests into the output directory, diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py index c1efc1c3d..0cdf8b8db 100644 --- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -16,7 +16,7 @@ The BuildReader contains basic logic for traversing a tree of mozbuild files. It does this by examining specific variables populated during execution. """ -from __future__ import absolute_import, print_function, unicode_literals + import ast import inspect @@ -81,12 +81,13 @@ from .context import ( from mozbuild.base import ExecutionSummary from concurrent.futures.process import ProcessPoolExecutor +from functools import reduce if sys.version_info.major == 2: - text_type = unicode - type_type = types.TypeType + text_type = str + type_type = type else: text_type = str type_type = type @@ -127,7 +128,7 @@ class EmptyConfig(object): self.substs = self.PopulateOnGetDict(EmptyValue, substs or self.default_substs) udict = {} - for k, v in self.substs.items(): + for k, v in list(self.substs.items()): if isinstance(v, str): udict[k.decode('utf-8')] = v.decode('utf-8') else: @@ -311,7 +312,7 @@ class MozbuildSandbox(Sandbox): raise Exception('`template` is a function decorator. You must ' 'use it as `@template` preceding a function declaration.') - name = func.func_name + name = func.__name__ if name in self.templates: raise KeyError( @@ -390,7 +391,7 @@ class MozbuildSandbox(Sandbox): klass = self._context.__class__ self._context.__class__ = TemplateContext # The sandbox will do all the necessary checks for these merges. - for key, value in context.items(): + for key, value in list(context.items()): if isinstance(value, dict): self[key].update(value) elif isinstance(value, (list, HierarchicalStringList)): @@ -407,12 +408,14 @@ class MozbuildSandbox(Sandbox): class TemplateFunction(object): def __init__(self, func, sandbox): - self.path = func.func_code.co_filename - self.name = func.func_name + self.path = func.__code__.co_filename + self.name = func.__name__ - code = func.func_code + code = func.__code__ firstlineno = code.co_firstlineno lines = sandbox._current_source.splitlines(True) + if len(lines) and isinstance(lines[0], bytes): + lines = [l.decode('utf-8') for l in lines] lines = inspect.getblock(lines[firstlineno - 1:]) # The code lines we get out of inspect.getsourcelines look like @@ -430,7 +433,7 @@ class TemplateFunction(object): # actually never calls __getitem__ and __setitem__, so we need to # modify the AST so that accesses to globals are properly directed # to a dict. - self._global_name = b'_data' # AST wants str for this, not unicode + self._global_name = '_data' # In case '_data' is a name used for a variable in the function code, # prepend more underscores until we find an unused name. while (self._global_name in code.co_names or @@ -449,8 +452,8 @@ class TemplateFunction(object): compile(func_ast, self.path, 'exec'), glob, self.name, - func.func_defaults, - func.func_closure, + func.__defaults__, + func.__closure__, ) func() @@ -464,11 +467,11 @@ class TemplateFunction(object): '__builtins__': sandbox._builtins } func = types.FunctionType( - self._func.func_code, + self._func.__code__, glob, self.name, - self._func.func_defaults, - self._func.func_closure + self._func.__defaults__, + self._func.__closure__ ) sandbox.exec_function(func, args, kwargs, self.path, becomes_current_path=False) @@ -484,7 +487,7 @@ class TemplateFunction(object): def visit_Str(self, node): # String nodes we got from the AST parser are str, but we want # unicode literals everywhere, so transform them. - node.s = unicode(node.s) + node.s = str(node.s) return node def visit_Name(self, node): @@ -617,7 +620,7 @@ class BuildReaderError(Exception): for l in traceback.format_exception(type(self.other), self.other, self.trace): - s.write(unicode(l)) + s.write(str(l)) return s.getvalue() @@ -767,7 +770,7 @@ class BuildReaderError(Exception): s.write(' %s\n' % inner.args[2]) s.write('\n') close_matches = difflib.get_close_matches(inner.args[2], - VARIABLES.keys(), 2) + list(VARIABLES.keys()), 2) if close_matches: s.write('Maybe you meant %s?\n' % ' or '.join(close_matches)) s.write('\n') @@ -1152,7 +1155,7 @@ class BuildReader(object): context) non_unified_sources.add(source) action_overrides = {} - for action, script in gyp_dir.action_overrides.iteritems(): + for action, script in gyp_dir.action_overrides.items(): action_overrides[action] = SourcePath(context, script) gyp_processor = GypProcessor(context.config, @@ -1188,7 +1191,7 @@ class BuildReader(object): recurse_info[d][key] = dict(sandbox.metadata[key]) - for path, child_metadata in recurse_info.items(): + for path, child_metadata in list(recurse_info.items()): child_path = path.join('moz.build').full_path # Ensure we don't break out of the topsrcdir. We don't do realpath @@ -1279,7 +1282,7 @@ class BuildReader(object): # There is room to improve this code (and the code in # _find_relevant_mozbuilds) to better handle multiple files in the same # directory. Bug 1136966 tracks. - for path, mbpaths in relevants.items(): + for path, mbpaths in list(relevants.items()): path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths] for i, mbpath in enumerate(mbpaths[0:-1]): @@ -1316,7 +1319,7 @@ class BuildReader(object): all_contexts.append(context) result = {} - for path, paths in path_mozbuilds.items(): + for path, paths in list(path_mozbuilds.items()): result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), []) return result, all_contexts @@ -1356,7 +1359,7 @@ class BuildReader(object): r = {} - for path, ctxs in paths.items(): + for path, ctxs in list(paths.items()): # Should be normalized by read_relevant_mozbuilds. assert '\\' not in path diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py index b2090802e..6d94291ea 100644 --- a/python/mozbuild/mozbuild/frontend/sandbox.py +++ b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -17,7 +17,7 @@ KeyError are machine parseable. This machine-friendly data is used to present user-friendly error messages in the case of errors. """ -from __future__ import absolute_import, unicode_literals + import os import sys @@ -112,6 +112,7 @@ class Sandbox(dict): 'int': int, 'set': set, 'tuple': tuple, + 'str': str, }) def __init__(self, context, finder=default_finder): diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py index 47a2eff63..96aea63ce 100644 --- a/python/mozbuild/mozbuild/jar.py +++ b/python/mozbuild/mozbuild/jar.py @@ -8,7 +8,7 @@ processing jar.mn files. See the documentation for jar.mn on MDC for further details on the format. ''' -from __future__ import absolute_import + import sys import os @@ -17,7 +17,7 @@ import re import logging from time import localtime from MozZipFile import ZipFile -from cStringIO import StringIO +from io import StringIO from collections import defaultdict from mozbuild.preprocessor import Preprocessor @@ -302,9 +302,9 @@ class JarMaker(object): '''updateManifest replaces the % in the chrome registration entries with the given chrome base path, and updates the given manifest file. ''' - myregister = dict.fromkeys(map(lambda s: s.replace('%', - chromebasepath), register)) - addEntriesToListFile(manifestPath, myregister.iterkeys()) + myregister = dict.fromkeys([s.replace('%', + chromebasepath) for s in register]) + addEntriesToListFile(manifestPath, iter(myregister.keys())) def makeJar(self, infile, jardir): '''makeJar is the main entry point to JarMaker. @@ -322,7 +322,7 @@ class JarMaker(object): elif self.relativesrcdir: self.localedirs = \ self.generateLocaleDirs(self.relativesrcdir) - if isinstance(infile, basestring): + if isinstance(infile, str): logging.info('processing ' + infile) self.sourcedirs.append(_normpath(os.path.dirname(infile))) pp = self.pp.clone() diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py index fcd45bed2..f77c5d2c3 100644 --- a/python/mozbuild/mozbuild/makeutil.py +++ b/python/mozbuild/mozbuild/makeutil.py @@ -2,11 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + import os import re -from types import StringTypes from collections import Iterable @@ -66,7 +65,7 @@ class _SimpleOrderedSet(object): self._list = [] self._set = set() - def __nonzero__(self): + def __bool__(self): return bool(self._set) def __iter__(self): @@ -103,19 +102,19 @@ class Rule(object): def add_targets(self, targets): '''Add additional targets to the rule.''' - assert isinstance(targets, Iterable) and not isinstance(targets, StringTypes) + assert isinstance(targets, Iterable) and not isinstance(targets, str) self._targets.update(targets) return self def add_dependencies(self, deps): '''Add dependencies to the rule.''' - assert isinstance(deps, Iterable) and not isinstance(deps, StringTypes) + assert isinstance(deps, Iterable) and not isinstance(deps, str) self._dependencies.update(deps) return self def add_commands(self, commands): '''Add commands to the rule.''' - assert isinstance(commands, Iterable) and not isinstance(commands, StringTypes) + assert isinstance(commands, Iterable) and not isinstance(commands, str) self._commands.extend(commands) return self @@ -139,13 +138,16 @@ class Rule(object): ''' if not self._targets: return - fh.write('%s:' % ' '.join(self._targets)) + wstring = '%s:' % ' '.join(self._targets) if self._dependencies: - fh.write(' %s' % ' '.join(self.dependencies())) - fh.write('\n') + wstring += ' %s' % ' '.join(self.dependencies()) + wstring += '\n' for cmd in self._commands: - fh.write('\t%s\n' % cmd) - + wstring += '\t%s\n' % cmd + try: + fh.write(wstring.encode('utf-8')) + except TypeError: + fh.write(wstring) # colon followed by anything except a slash (Windows path detection) _depfilesplitter = re.compile(r':(?![\\/])') diff --git a/python/mozbuild/mozbuild/mozinfo.py b/python/mozbuild/mozbuild/mozinfo.py index 7e7ad1b2a..f03f20ba3 100755 --- a/python/mozbuild/mozbuild/mozinfo.py +++ b/python/mozbuild/mozbuild/mozinfo.py @@ -5,7 +5,7 @@ # This module produces a JSON file that provides basic build info and # configuration metadata. -from __future__ import absolute_import + import os import re @@ -33,7 +33,7 @@ def build_dict(config, env=os.environ): d['mozconfig'] = config.mozconfig # os - o = substs["OS_TARGET"] + o = str(substs["OS_TARGET"]) known_os = {"Linux": "linux", "WINNT": "win", "Darwin": "mac", @@ -148,7 +148,7 @@ def write_mozinfo(file, config, env=os.environ): and what keys are produced. """ build_conf = build_dict(config, env) - if isinstance(file, basestring): - file = open(file, 'wb') + if isinstance(file, str): + file = open(file, 'w') json.dump(build_conf, file, sort_keys=True, indent=4) diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py index 6780b8b72..19e59884e 100644 --- a/python/mozbuild/mozbuild/preprocessor.py +++ b/python/mozbuild/mozbuild/preprocessor.py @@ -27,7 +27,8 @@ import os import re from optparse import OptionParser import errno -from makeutil import Makefile +from .makeutil import Makefile +from functools import reduce # hack around win32 mangling our line endings # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443 @@ -230,7 +231,7 @@ class Expression: def __repr__(self): return self.value.__repr__() - class ParseError(StandardError): + class ParseError(Exception): """ Error raised when parsing fails. It has two members, offset and content, which give the offset of the @@ -278,7 +279,7 @@ class Preprocessor: self.context = Context() for k,v in {'FILE': '', 'LINE': 0, - 'DIRECTORY': os.path.abspath('.')}.iteritems(): + 'DIRECTORY': os.path.abspath('.')}.items(): self.context[k] = v self.actionLevel = 0 self.disableLevel = 0 @@ -292,21 +293,21 @@ class Preprocessor: self.cmds = {} for cmd, level in {'define': 0, 'undef': 0, - 'if': sys.maxint, - 'ifdef': sys.maxint, - 'ifndef': sys.maxint, + 'if': sys.maxsize, + 'ifdef': sys.maxsize, + 'ifndef': sys.maxsize, 'else': 1, 'elif': 1, 'elifdef': 1, 'elifndef': 1, - 'endif': sys.maxint, + 'endif': sys.maxsize, 'expand': 0, 'literal': 0, 'filter': 0, 'unfilter': 0, 'include': 0, 'includesubst': 0, - 'error': 0}.iteritems(): + 'error': 0}.items(): self.cmds[cmd] = (level, getattr(self, 'do_' + cmd)) self.out = sys.stdout self.setMarker(marker) @@ -434,7 +435,7 @@ class Preprocessor: filteredLine = self.applyFilters(aLine) if filteredLine != aLine: self.actionLevel = 2 - self.out.write(filteredLine) + self.out.write(filteredLine.encode('utf-8')) def handleCommandLine(self, args, defaultToStdin = False): """ @@ -468,7 +469,7 @@ class Preprocessor: raise Preprocessor.Error(self, "--depend doesn't work with stdout", None) try: - from makeutil import Makefile + from .makeutil import Makefile except: raise Preprocessor.Error(self, "--depend requires the " "mozbuild.makeutil module", None) @@ -683,7 +684,7 @@ class Preprocessor: current = dict(self.filters) for f in filters: current[f] = getattr(self, 'filter_' + f) - filterNames = current.keys() + filterNames = list(current.keys()) filterNames.sort() self.filters = [(fn, current[fn]) for fn in filterNames] return @@ -693,7 +694,7 @@ class Preprocessor: for f in filters: if f in current: del current[f] - filterNames = current.keys() + filterNames = list(current.keys()) filterNames.sort() self.filters = [(fn, current[fn]) for fn in filterNames] return @@ -738,7 +739,7 @@ class Preprocessor: args can either be a file name, or a file-like object. Files should be opened, and will be closed after processing. """ - isName = type(args) == str or type(args) == unicode + isName = type(args) == str or type(args) == str oldCheckLineNumbers = self.checkLineNumbers self.checkLineNumbers = False if isName: diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py index 185a970ee..c0c15f8f2 100644 --- a/python/mozbuild/mozbuild/shellutil.py +++ b/python/mozbuild/mozbuild/shellutil.py @@ -15,7 +15,7 @@ def _tokens2re(**tokens): # which matches the pattern and captures it in a named match group. # The group names and patterns are given as arguments. all_tokens = '|'.join('(?P<%s>%s)' % (name, value) - for name, value in tokens.iteritems()) + for name, value in tokens.items()) nonescaped = r'(?` is unused." % mozpath.join(test_data_path, 'moz.configure')) @@ -257,7 +257,7 @@ class TestLint(unittest.TestCase): '''): self.lint_test() - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "%s:9: The dependency on `qux` is unused." % mozpath.join(test_data_path, 'moz.configure')) diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py index b3342e268..7a26e9e09 100644 --- a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + from mozunit import main from mozpack import path as mozpath @@ -27,10 +27,10 @@ class TestMozConfigure(BaseConfigureTest): shell = mozpath.abspath('/bin/sh') return result.replace('CONFIG_SHELL=%s ' % shell, '') - self.assertEquals('--enable-application=browser', + self.assertEqual('--enable-application=browser', get_value_for(['--enable-application=browser'])) - self.assertEquals('--enable-application=browser ' + self.assertEqual('--enable-application=browser ' 'MOZ_VTUNE=1', get_value_for(['--enable-application=browser', 'MOZ_VTUNE=1'])) @@ -39,25 +39,25 @@ class TestMozConfigure(BaseConfigureTest): environ={'MOZ_VTUNE': '1'}, mozconfig='ac_add_options --enable-project=js') - self.assertEquals('--enable-project=js MOZ_VTUNE=1', + self.assertEqual('--enable-project=js MOZ_VTUNE=1', value) # --disable-js-shell is the default, so it's filtered out. - self.assertEquals('--enable-application=browser', + self.assertEqual('--enable-application=browser', get_value_for(['--enable-application=browser', '--disable-js-shell'])) # Normally, --without-foo would be filtered out because that's the # default, but since it is a (fake) old-configure option, it always # appears. - self.assertEquals('--enable-application=browser --without-foo', + self.assertEqual('--enable-application=browser --without-foo', get_value_for(['--enable-application=browser', '--without-foo'])) - self.assertEquals('--enable-application=browser --with-foo', + self.assertEqual('--enable-application=browser --with-foo', get_value_for(['--enable-application=browser', '--with-foo'])) - self.assertEquals("--enable-application=browser '--with-foo=foo bar'", + self.assertEqual("--enable-application=browser '--with-foo=foo bar'", get_value_for(['--enable-application=browser', '--with-foo=foo bar'])) @@ -69,7 +69,7 @@ class TestMozConfigure(BaseConfigureTest): self.version = version def __call__(self, stdin, args): - this.assertEquals(args, ('-version',)) + this.assertEqual(args, ('-version',)) return 0, self.version, '' def check_nsis_version(version): @@ -84,13 +84,13 @@ class TestMozConfigure(BaseConfigureTest): with self.assertRaises(SystemExit) as e: check_nsis_version('v3.0a2') - self.assertEquals(check_nsis_version('v3.0b1'), '3.0b1') - self.assertEquals(check_nsis_version('v3.0b2'), '3.0b2') - self.assertEquals(check_nsis_version('v3.0rc1'), '3.0rc1') - self.assertEquals(check_nsis_version('v3.0'), '3.0') - self.assertEquals(check_nsis_version('v3.0-2'), '3.0') - self.assertEquals(check_nsis_version('v3.0.1'), '3.0') - self.assertEquals(check_nsis_version('v3.1'), '3.1') + self.assertEqual(check_nsis_version('v3.0b1'), '3.0b1') + self.assertEqual(check_nsis_version('v3.0b2'), '3.0b2') + self.assertEqual(check_nsis_version('v3.0rc1'), '3.0rc1') + self.assertEqual(check_nsis_version('v3.0'), '3.0') + self.assertEqual(check_nsis_version('v3.0-2'), '3.0') + self.assertEqual(check_nsis_version('v3.0.1'), '3.0') + self.assertEqual(check_nsis_version('v3.1'), '3.1') if __name__ == '__main__': diff --git a/python/mozbuild/mozbuild/test/configure/test_options.py b/python/mozbuild/mozbuild/test/configure/test_options.py index 9defccb2c..330ce3b1f 100644 --- a/python/mozbuild/mozbuild/test/configure/test_options.py +++ b/python/mozbuild/mozbuild/test/configure/test_options.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import unittest @@ -28,139 +28,139 @@ class Option(Option): class TestOption(unittest.TestCase): def test_option(self): option = Option('--option') - self.assertEquals(option.prefix, '') - self.assertEquals(option.name, 'option') - self.assertEquals(option.env, None) + self.assertEqual(option.prefix, '') + self.assertEqual(option.name, 'option') + self.assertEqual(option.env, None) self.assertFalse(option.default) option = Option('--enable-option') - self.assertEquals(option.prefix, 'enable') - self.assertEquals(option.name, 'option') - self.assertEquals(option.env, None) + self.assertEqual(option.prefix, 'enable') + self.assertEqual(option.name, 'option') + self.assertEqual(option.env, None) self.assertFalse(option.default) option = Option('--disable-option') - self.assertEquals(option.prefix, 'disable') - self.assertEquals(option.name, 'option') - self.assertEquals(option.env, None) + self.assertEqual(option.prefix, 'disable') + self.assertEqual(option.name, 'option') + self.assertEqual(option.env, None) self.assertTrue(option.default) option = Option('--with-option') - self.assertEquals(option.prefix, 'with') - self.assertEquals(option.name, 'option') - self.assertEquals(option.env, None) + self.assertEqual(option.prefix, 'with') + self.assertEqual(option.name, 'option') + self.assertEqual(option.env, None) self.assertFalse(option.default) option = Option('--without-option') - self.assertEquals(option.prefix, 'without') - self.assertEquals(option.name, 'option') - self.assertEquals(option.env, None) + self.assertEqual(option.prefix, 'without') + self.assertEqual(option.name, 'option') + self.assertEqual(option.env, None) self.assertTrue(option.default) option = Option('--without-option-foo', env='MOZ_OPTION') - self.assertEquals(option.env, 'MOZ_OPTION') + self.assertEqual(option.env, 'MOZ_OPTION') option = Option(env='MOZ_OPTION') - self.assertEquals(option.prefix, '') - self.assertEquals(option.name, None) - self.assertEquals(option.env, 'MOZ_OPTION') + self.assertEqual(option.prefix, '') + self.assertEqual(option.name, None) + self.assertEqual(option.env, 'MOZ_OPTION') self.assertFalse(option.default) with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=0, default=('a',)) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=1, default=()) - self.assertEquals( + self.assertEqual( e.exception.message, 'default must be a bool, a string or a tuple of strings') with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=1, default=True) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=1, default=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=2, default=()) - self.assertEquals( + self.assertEqual( e.exception.message, 'default must be a bool, a string or a tuple of strings') with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=2, default=True) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=2, default=('a',)) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs='?', default=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs='+', default=()) - self.assertEquals( + self.assertEqual( e.exception.message, 'default must be a bool, a string or a tuple of strings') with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs='+', default=True) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") # --disable options with a nargs value that requires at least one # argument need to be given a default. with self.assertRaises(InvalidOptionError) as e: Option('--disable-option', nargs=1) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") with self.assertRaises(InvalidOptionError) as e: Option('--disable-option', nargs='+') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") # Test nargs inference from default value option = Option('--with-foo', default=True) - self.assertEquals(option.nargs, 0) + self.assertEqual(option.nargs, 0) option = Option('--with-foo', default=False) - self.assertEquals(option.nargs, 0) + self.assertEqual(option.nargs, 0) option = Option('--with-foo', default='a') - self.assertEquals(option.nargs, '?') + self.assertEqual(option.nargs, '?') option = Option('--with-foo', default=('a',)) - self.assertEquals(option.nargs, '?') + self.assertEqual(option.nargs, '?') option = Option('--with-foo', default=('a', 'b')) - self.assertEquals(option.nargs, '*') + self.assertEqual(option.nargs, '*') option = Option(env='FOO', default=True) - self.assertEquals(option.nargs, 0) + self.assertEqual(option.nargs, 0) option = Option(env='FOO', default=False) - self.assertEquals(option.nargs, 0) + self.assertEqual(option.nargs, 0) option = Option(env='FOO', default='a') - self.assertEquals(option.nargs, '?') + self.assertEqual(option.nargs, '?') option = Option(env='FOO', default=('a',)) - self.assertEquals(option.nargs, '?') + self.assertEqual(option.nargs, '?') option = Option(env='FOO', default=('a', 'b')) - self.assertEquals(option.nargs, '*') + self.assertEqual(option.nargs, '*') def test_option_option(self): for option in ( @@ -170,70 +170,70 @@ class TestOption(unittest.TestCase): '--with-option', '--without-option', ): - self.assertEquals(Option(option).option, option) - self.assertEquals(Option(option, env='FOO').option, option) + self.assertEqual(Option(option).option, option) + self.assertEqual(Option(option, env='FOO').option, option) opt = Option(option, default=False) - self.assertEquals(opt.option, + self.assertEqual(opt.option, option.replace('-disable-', '-enable-') .replace('-without-', '-with-')) opt = Option(option, default=True) - self.assertEquals(opt.option, + self.assertEqual(opt.option, option.replace('-enable-', '-disable-') .replace('-with-', '-without-')) - self.assertEquals(Option(env='FOO').option, 'FOO') + self.assertEqual(Option(env='FOO').option, 'FOO') def test_option_choices(self): with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=3, choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Not enough `choices` for `nargs`') with self.assertRaises(InvalidOptionError) as e: Option('--without-option', nargs=1, choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'A `default` must be given along with `choices`') with self.assertRaises(InvalidOptionError) as e: Option('--without-option', nargs='+', choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'A `default` must be given along with `choices`') with self.assertRaises(InvalidOptionError) as e: Option('--without-option', default='c', choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The `default` value must be one of 'a', 'b'") with self.assertRaises(InvalidOptionError) as e: Option('--without-option', default=('a', 'c',), choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The `default` value must be one of 'a', 'b'") with self.assertRaises(InvalidOptionError) as e: Option('--without-option', default=('c',), choices=('a', 'b')) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The `default` value must be one of 'a', 'b'") option = Option('--with-option', nargs='+', choices=('a', 'b')) with self.assertRaises(InvalidOptionError) as e: option.get_value('--with-option=c') - self.assertEquals(e.exception.message, "'c' is not one of 'a', 'b'") + self.assertEqual(e.exception.message, "'c' is not one of 'a', 'b'") value = option.get_value('--with-option=b,a') self.assertTrue(value) - self.assertEquals(PositiveOptionValue(('b', 'a')), value) + self.assertEqual(PositiveOptionValue(('b', 'a')), value) option = Option('--without-option', nargs='*', default='a', choices=('a', 'b')) with self.assertRaises(InvalidOptionError) as e: option.get_value('--with-option=c') - self.assertEquals(e.exception.message, "'c' is not one of 'a', 'b'") + self.assertEqual(e.exception.message, "'c' is not one of 'a', 'b'") value = option.get_value('--with-option=b,a') self.assertTrue(value) - self.assertEquals(PositiveOptionValue(('b', 'a')), value) + self.assertEqual(PositiveOptionValue(('b', 'a')), value) # Test nargs inference from choices option = Option('--with-option', choices=('a', 'b')) @@ -244,37 +244,37 @@ class TestOption(unittest.TestCase): choices=('a', 'b', 'c', 'd')) value = option.get_value('--with-option=+d') - self.assertEquals(PositiveOptionValue(('b', 'c', 'd')), value) + self.assertEqual(PositiveOptionValue(('b', 'c', 'd')), value) value = option.get_value('--with-option=-b') - self.assertEquals(PositiveOptionValue(('c',)), value) + self.assertEqual(PositiveOptionValue(('c',)), value) value = option.get_value('--with-option=-b,+d') - self.assertEquals(PositiveOptionValue(('c','d')), value) + self.assertEqual(PositiveOptionValue(('c','d')), value) # Adding something that is in the default is fine value = option.get_value('--with-option=+b') - self.assertEquals(PositiveOptionValue(('b', 'c')), value) + self.assertEqual(PositiveOptionValue(('b', 'c')), value) # Removing something that is not in the default is fine, as long as it # is one of the choices value = option.get_value('--with-option=-a') - self.assertEquals(PositiveOptionValue(('b', 'c')), value) + self.assertEqual(PositiveOptionValue(('b', 'c')), value) with self.assertRaises(InvalidOptionError) as e: option.get_value('--with-option=-e') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "'e' is not one of 'a', 'b', 'c', 'd'") # Other "not a choice" errors. with self.assertRaises(InvalidOptionError) as e: option.get_value('--with-option=+e') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "'e' is not one of 'a', 'b', 'c', 'd'") with self.assertRaises(InvalidOptionError) as e: option.get_value('--with-option=e') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "'e' is not one of 'a', 'b', 'c', 'd'") def test_option_value_compare(self): @@ -294,7 +294,7 @@ class TestOption(unittest.TestCase): # For usability reasons, we raise TypeError when attempting to compare # against a non-tuple. - with self.assertRaisesRegexp(TypeError, 'cannot compare a'): + with self.assertRaisesRegex(TypeError, 'cannot compare a'): val == 'foo' # But we allow empty option values to compare otherwise we can't @@ -313,36 +313,36 @@ class TestOption(unittest.TestCase): def test_option_value_format(self): val = PositiveOptionValue() - self.assertEquals('--with-value', val.format('--with-value')) - self.assertEquals('--with-value', val.format('--without-value')) - self.assertEquals('--enable-value', val.format('--enable-value')) - self.assertEquals('--enable-value', val.format('--disable-value')) - self.assertEquals('--value', val.format('--value')) - self.assertEquals('VALUE=1', val.format('VALUE')) + self.assertEqual('--with-value', val.format('--with-value')) + self.assertEqual('--with-value', val.format('--without-value')) + self.assertEqual('--enable-value', val.format('--enable-value')) + self.assertEqual('--enable-value', val.format('--disable-value')) + self.assertEqual('--value', val.format('--value')) + self.assertEqual('VALUE=1', val.format('VALUE')) val = PositiveOptionValue(('a',)) - self.assertEquals('--with-value=a', val.format('--with-value')) - self.assertEquals('--with-value=a', val.format('--without-value')) - self.assertEquals('--enable-value=a', val.format('--enable-value')) - self.assertEquals('--enable-value=a', val.format('--disable-value')) - self.assertEquals('--value=a', val.format('--value')) - self.assertEquals('VALUE=a', val.format('VALUE')) + self.assertEqual('--with-value=a', val.format('--with-value')) + self.assertEqual('--with-value=a', val.format('--without-value')) + self.assertEqual('--enable-value=a', val.format('--enable-value')) + self.assertEqual('--enable-value=a', val.format('--disable-value')) + self.assertEqual('--value=a', val.format('--value')) + self.assertEqual('VALUE=a', val.format('VALUE')) val = PositiveOptionValue(('a', 'b')) - self.assertEquals('--with-value=a,b', val.format('--with-value')) - self.assertEquals('--with-value=a,b', val.format('--without-value')) - self.assertEquals('--enable-value=a,b', val.format('--enable-value')) - self.assertEquals('--enable-value=a,b', val.format('--disable-value')) - self.assertEquals('--value=a,b', val.format('--value')) - self.assertEquals('VALUE=a,b', val.format('VALUE')) + self.assertEqual('--with-value=a,b', val.format('--with-value')) + self.assertEqual('--with-value=a,b', val.format('--without-value')) + self.assertEqual('--enable-value=a,b', val.format('--enable-value')) + self.assertEqual('--enable-value=a,b', val.format('--disable-value')) + self.assertEqual('--value=a,b', val.format('--value')) + self.assertEqual('VALUE=a,b', val.format('VALUE')) val = NegativeOptionValue() - self.assertEquals('--without-value', val.format('--with-value')) - self.assertEquals('--without-value', val.format('--without-value')) - self.assertEquals('--disable-value', val.format('--enable-value')) - self.assertEquals('--disable-value', val.format('--disable-value')) - self.assertEquals('', val.format('--value')) - self.assertEquals('VALUE=', val.format('VALUE')) + self.assertEqual('--without-value', val.format('--with-value')) + self.assertEqual('--without-value', val.format('--without-value')) + self.assertEqual('--disable-value', val.format('--enable-value')) + self.assertEqual('--disable-value', val.format('--disable-value')) + self.assertEqual('', val.format('--value')) + self.assertEqual('VALUE=', val.format('VALUE')) def test_option_value(self, name='option', nargs=0, default=None): disabled = name.startswith(('disable-', 'without-')) @@ -359,28 +359,28 @@ class TestOption(unittest.TestCase): if nargs in (0, '?', '*') or disabled: value = option.get_value('--%s' % name, 'option') - self.assertEquals(value, posOptionValue()) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, posOptionValue()) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s' % name) if nargs == 1: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes 1 value' % name) elif nargs == '+': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes 1 or more values' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes 2 values' % name) value = option.get_value('') - self.assertEquals(value, defaultValue) - self.assertEquals(value.origin, 'default') + self.assertEqual(value, defaultValue) + self.assertEqual(value.origin, 'default') value = option.get_value(None) - self.assertEquals(value, defaultValue) - self.assertEquals(value.origin, 'default') + self.assertEqual(value, defaultValue) + self.assertEqual(value.origin, 'default') with self.assertRaises(AssertionError): value = option.get_value('MOZ_OPTION=', 'environment') @@ -393,47 +393,47 @@ class TestOption(unittest.TestCase): if nargs in (1, '?', '*', '+') and not disabled: value = option.get_value('--%s=' % name, 'option') - self.assertEquals(value, PositiveOptionValue(('',))) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, PositiveOptionValue(('',))) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s=' % name) if disabled: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Cannot pass a value to --%s' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes %d values' % (name, nargs)) if nargs in (1, '?', '*', '+') and not disabled: value = option.get_value('--%s=foo' % name, 'option') - self.assertEquals(value, PositiveOptionValue(('foo',))) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, PositiveOptionValue(('foo',))) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s=foo' % name) if disabled: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Cannot pass a value to --%s' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes %d values' % (name, nargs)) if nargs in (2, '*', '+') and not disabled: value = option.get_value('--%s=foo,bar' % name, 'option') - self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s=foo,bar' % name, 'option') if disabled: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Cannot pass a value to --%s' % name) elif nargs == '?': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes 0 or 1 values' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes %d value%s' % (name, nargs, 's' if nargs != 1 else '')) @@ -441,59 +441,59 @@ class TestOption(unittest.TestCase): default=default) if nargs in (0, '?', '*') or disabled: value = option.get_value('--%s' % name, 'option') - self.assertEquals(value, posOptionValue()) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, posOptionValue()) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s' % name) if disabled: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Cannot pass a value to --%s' % name) elif nargs == '+': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes 1 or more values' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes %d value%s' % (name, nargs, 's' if nargs != 1 else '')) value = option.get_value('') - self.assertEquals(value, defaultValue) - self.assertEquals(value.origin, 'default') + self.assertEqual(value, defaultValue) + self.assertEqual(value.origin, 'default') value = option.get_value(None) - self.assertEquals(value, defaultValue) - self.assertEquals(value.origin, 'default') + self.assertEqual(value, defaultValue) + self.assertEqual(value.origin, 'default') value = option.get_value('MOZ_OPTION=', 'environment') - self.assertEquals(value, NegativeOptionValue()) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, NegativeOptionValue()) + self.assertEqual(value.origin, 'environment') if nargs in (0, '?', '*'): value = option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(value, PositiveOptionValue()) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue()) + self.assertEqual(value.origin, 'environment') elif nargs in (1, '+'): value = option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(value, PositiveOptionValue(('1',))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('1',))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(e.exception.message, 'MOZ_OPTION takes 2 values') + self.assertEqual(e.exception.message, 'MOZ_OPTION takes 2 values') if nargs in (1, '?', '*', '+') and not disabled: value = option.get_value('--%s=' % name, 'option') - self.assertEquals(value, PositiveOptionValue(('',))) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, PositiveOptionValue(('',))) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s=' % name, 'option') if disabled: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'Cannot pass a value to --%s' % name) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s takes %d values' % (name, nargs)) with self.assertRaises(AssertionError): @@ -501,26 +501,26 @@ class TestOption(unittest.TestCase): if nargs in (1, '?', '*', '+'): value = option.get_value('MOZ_OPTION=foo', 'environment') - self.assertEquals(value, PositiveOptionValue(('foo',))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('foo',))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('MOZ_OPTION=foo', 'environment') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes %d values' % nargs) if nargs in (2, '*', '+'): value = option.get_value('MOZ_OPTION=foo,bar', 'environment') - self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('MOZ_OPTION=foo,bar', 'environment') if nargs == '?': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes 0 or 1 values') else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes %d value%s' % (nargs, 's' if nargs != 1 else '')) @@ -532,26 +532,26 @@ class TestOption(unittest.TestCase): env_option.get_value('--%s' % name) value = env_option.get_value('') - self.assertEquals(value, defaultValue) - self.assertEquals(value.origin, 'default') + self.assertEqual(value, defaultValue) + self.assertEqual(value.origin, 'default') value = env_option.get_value('MOZ_OPTION=', 'environment') - self.assertEquals(value, negOptionValue()) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, negOptionValue()) + self.assertEqual(value.origin, 'environment') if nargs in (0, '?', '*'): value = env_option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(value, posOptionValue()) + self.assertEqual(value, posOptionValue()) self.assertTrue(value) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value.origin, 'environment') elif nargs in (1, '+'): value = env_option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(value, PositiveOptionValue(('1',))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('1',))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: env_option.get_value('MOZ_OPTION=1', 'environment') - self.assertEquals(e.exception.message, 'MOZ_OPTION takes 2 values') + self.assertEqual(e.exception.message, 'MOZ_OPTION takes 2 values') with self.assertRaises(AssertionError) as e: env_option.get_value('--%s' % name) @@ -561,26 +561,26 @@ class TestOption(unittest.TestCase): if nargs in (1, '?', '*', '+'): value = env_option.get_value('MOZ_OPTION=foo', 'environment') - self.assertEquals(value, PositiveOptionValue(('foo',))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('foo',))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: env_option.get_value('MOZ_OPTION=foo', 'environment') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes %d values' % nargs) if nargs in (2, '*', '+'): value = env_option.get_value('MOZ_OPTION=foo,bar', 'environment') - self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) - self.assertEquals(value.origin, 'environment') + self.assertEqual(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEqual(value.origin, 'environment') else: with self.assertRaises(InvalidOptionError) as e: env_option.get_value('MOZ_OPTION=foo,bar', 'environment') if nargs == '?': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes 0 or 1 values') else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, 'MOZ_OPTION takes %d value%s' % (nargs, 's' if nargs != 1 else '')) @@ -592,28 +592,28 @@ class TestOption(unittest.TestCase): default=default) value = option.get_value('--%s-option' % disable, 'option') - self.assertEquals(value, NegativeOptionValue()) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, NegativeOptionValue()) + self.assertEqual(value.origin, 'option') option = self.test_option_value('%s-option' % disable, nargs=nargs, default=default) if nargs in (0, '?', '*'): value = option.get_value('--%s-option' % enable, 'option') - self.assertEquals(value, PositiveOptionValue()) - self.assertEquals(value.origin, 'option') + self.assertEqual(value, PositiveOptionValue()) + self.assertEqual(value.origin, 'option') else: with self.assertRaises(InvalidOptionError) as e: option.get_value('--%s-option' % enable, 'option') if nargs == 1: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s-option takes 1 value' % enable) elif nargs == '+': - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s-option takes 1 or more values' % enable) else: - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, '--%s-option takes 2 values' % enable) def test_option_value_with(self): @@ -622,12 +622,12 @@ class TestOption(unittest.TestCase): def test_option_value_invalid_nargs(self): with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs='foo') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "nargs must be a positive integer, '?', '*' or '+'") with self.assertRaises(InvalidOptionError) as e: Option('--option', nargs=-2) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "nargs must be a positive integer, '?', '*' or '+'") def test_option_value_nargs_1(self): @@ -638,7 +638,7 @@ class TestOption(unittest.TestCase): # A default is required with self.assertRaises(InvalidOptionError) as e: Option('--disable-option', nargs=1) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") def test_option_value_nargs_2(self): @@ -649,7 +649,7 @@ class TestOption(unittest.TestCase): # A default is required with self.assertRaises(InvalidOptionError) as e: Option('--disable-option', nargs=2) - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") def test_option_value_nargs_0_or_1(self): @@ -676,7 +676,7 @@ class TestOption(unittest.TestCase): # A default is required with self.assertRaises(InvalidOptionError) as e: Option('--disable-option', nargs='+') - self.assertEquals(e.exception.message, + self.assertEqual(e.exception.message, "The given `default` doesn't satisfy `nargs`") @@ -684,21 +684,21 @@ class TestCommandLineHelper(unittest.TestCase): def test_basic(self): helper = CommandLineHelper({}, ['cmd', '--foo', '--bar']) - self.assertEquals(['--foo', '--bar'], list(helper)) + self.assertEqual(['--foo', '--bar'], list(helper)) helper.add('--enable-qux') - self.assertEquals(['--foo', '--bar', '--enable-qux'], list(helper)) + self.assertEqual(['--foo', '--bar', '--enable-qux'], list(helper)) value, option = helper.handle(Option('--bar')) - self.assertEquals(['--foo', '--enable-qux'], list(helper)) - self.assertEquals(PositiveOptionValue(), value) - self.assertEquals('--bar', option) + self.assertEqual(['--foo', '--enable-qux'], list(helper)) + self.assertEqual(PositiveOptionValue(), value) + self.assertEqual('--bar', option) value, option = helper.handle(Option('--baz')) - self.assertEquals(['--foo', '--enable-qux'], list(helper)) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals(None, option) + self.assertEqual(['--foo', '--enable-qux'], list(helper)) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual(None, option) with self.assertRaises(AssertionError): CommandLineHelper({}, ['--foo', '--bar']) @@ -707,89 +707,89 @@ class TestCommandLineHelper(unittest.TestCase): foo = Option('--with-foo', nargs='*') helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--with-foo=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--with-foo=a,b', option) helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b', '--without-foo']) value, option = helper.handle(foo) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--without-foo', option) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--without-foo', option) helper = CommandLineHelper({}, ['cmd', '--without-foo', '--with-foo=a,b']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--with-foo=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--with-foo=a,b', option) foo = Option('--with-foo', env='FOO', nargs='*') helper = CommandLineHelper({'FOO': ''}, ['cmd', '--with-foo=a,b']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--with-foo=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--with-foo=a,b', option) helper = CommandLineHelper({'FOO': 'a,b'}, ['cmd', '--without-foo']) value, option = helper.handle(foo) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--without-foo', option) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--without-foo', option) helper = CommandLineHelper({'FOO': ''}, ['cmd', '--with-bar=a,b']) value, option = helper.handle(foo) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals('environment', value.origin) - self.assertEquals('FOO=', option) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual('environment', value.origin) + self.assertEqual('FOO=', option) helper = CommandLineHelper({'FOO': 'a,b'}, ['cmd', '--without-bar']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('environment', value.origin) - self.assertEquals('FOO=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('environment', value.origin) + self.assertEqual('FOO=a,b', option) helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b', 'FOO=']) value, option = helper.handle(foo) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('FOO=', option) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('FOO=', option) helper = CommandLineHelper({}, ['cmd', '--without-foo', 'FOO=a,b']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('FOO=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('FOO=a,b', option) helper = CommandLineHelper({}, ['cmd', 'FOO=', '--with-foo=a,b']) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b')), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--with-foo=a,b', option) + self.assertEqual(PositiveOptionValue(('a', 'b')), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--with-foo=a,b', option) helper = CommandLineHelper({}, ['cmd', 'FOO=a,b', '--without-foo']) value, option = helper.handle(foo) - self.assertEquals(NegativeOptionValue(), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--without-foo', option) + self.assertEqual(NegativeOptionValue(), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--without-foo', option) def test_extra_args(self): foo = Option('--with-foo', env='FOO', nargs='*') helper = CommandLineHelper({}, ['cmd']) helper.add('FOO=a,b,c', 'other-origin') value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) - self.assertEquals('other-origin', value.origin) - self.assertEquals('FOO=a,b,c', option) + self.assertEqual(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEqual('other-origin', value.origin) + self.assertEqual('FOO=a,b,c', option) helper = CommandLineHelper({}, ['cmd']) helper.add('FOO=a,b,c', 'other-origin') helper.add('--with-foo=a,b,c', 'other-origin') value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) - self.assertEquals('other-origin', value.origin) - self.assertEquals('--with-foo=a,b,c', option) + self.assertEqual(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEqual('other-origin', value.origin) + self.assertEqual('--with-foo=a,b,c', option) # Adding conflicting options is not allowed. helper = CommandLineHelper({}, ['cmd']) @@ -809,9 +809,9 @@ class TestCommandLineHelper(unittest.TestCase): # But adding the same is allowed. helper.add('FOO=a,b,c', 'other-origin') value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) - self.assertEquals('other-origin', value.origin) - self.assertEquals('FOO=a,b,c', option) + self.assertEqual(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEqual('other-origin', value.origin) + self.assertEqual('FOO=a,b,c', option) # The same rule as above applies when using the option form vs. the # variable form. But we can't detect it when .add is called. @@ -837,9 +837,9 @@ class TestCommandLineHelper(unittest.TestCase): helper.add('FOO=a,b,c', 'other-origin') helper.add('--with-foo=a,b,c', 'other-origin') value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) - self.assertEquals('other-origin', value.origin) - self.assertEquals('--with-foo=a,b,c', option) + self.assertEqual(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEqual('other-origin', value.origin) + self.assertEqual('--with-foo=a,b,c', option) # Conflicts are also not allowed against what is in the # environment/on the command line. @@ -869,19 +869,19 @@ class TestCommandLineHelper(unittest.TestCase): foo = Option('--foo', possible_origins=('command-line',)) value, option = helper.handle(foo) - self.assertEquals(PositiveOptionValue(), value) - self.assertEquals('command-line', value.origin) - self.assertEquals('--foo', option) + self.assertEqual(PositiveOptionValue(), value) + self.assertEqual('command-line', value.origin) + self.assertEqual('--foo', option) bar = Option('--bar', possible_origins=('mozconfig',)) - with self.assertRaisesRegexp(InvalidOptionError, + with self.assertRaisesRegex(InvalidOptionError, "--bar can not be set by command-line. Values are accepted from: mozconfig"): helper.handle(bar) baz = Option(env='BAZ', possible_origins=('implied',)) - with self.assertRaisesRegexp(InvalidOptionError, + with self.assertRaisesRegex(InvalidOptionError, "BAZ=1 can not be set by environment. Values are accepted from: implied"): helper.handle(baz) diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py index cb7ff709e..c339a32bf 100755 --- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py @@ -2,12 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import logging import os -from StringIO import StringIO +from io import StringIO from mozunit import main @@ -366,9 +366,9 @@ class BaseToolchainTest(BaseConfigureTest): compiler = sandbox._value_for(sandbox[var]) # Add var on both ends to make it clear which of the # variables is failing the test when that happens. - self.assertEquals((var, compiler), (var, result)) + self.assertEqual((var, compiler), (var, result)) except SystemExit: - self.assertEquals((var, result), + self.assertEqual((var, result), (var, self.out.getvalue().strip())) return @@ -407,7 +407,7 @@ class BaseToolchainTest(BaseConfigureTest): 'RUST_LIB_SUFFIX', 'OBJ_SUFFIX', ): - self.assertEquals('%s=%s' % (k, sandbox.get_config(k)), + self.assertEqual('%s=%s' % (k, sandbox.get_config(k)), '%s=%s' % (k, library_name_info[k])) @@ -584,7 +584,7 @@ class LinuxToolchainTest(BaseToolchainTest): # We'll try gcc and clang, but since there is no gcc (gcc-x.y doesn't # count), find clang. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) not in ('gcc', 'g++') } self.do_toolchain_test(paths, { @@ -619,7 +619,7 @@ class LinuxToolchainTest(BaseToolchainTest): # Even if there are gcc-x.y or clang-x.y compilers available, we # don't try them. This could be considered something to improve. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) not in ('gcc', 'g++', 'clang', 'clang++') } self.do_toolchain_test(paths, { @@ -800,7 +800,7 @@ class OSXToolchainTest(BaseToolchainTest): def test_not_gcc(self): # We won't pick GCC if it's the only thing available. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) not in ('clang', 'clang++') } self.do_toolchain_test(paths, { @@ -976,7 +976,7 @@ class WindowsToolchainTest(BaseToolchainTest): def test_clang_cl(self): # We'll pick clang-cl if msvc can't be found. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) != 'cl' } self.do_toolchain_test(paths, { @@ -987,7 +987,7 @@ class WindowsToolchainTest(BaseToolchainTest): def test_gcc(self): # We'll pick GCC if msvc and clang-cl can't be found. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) not in ('cl', 'clang-cl') } self.do_toolchain_test(paths, { @@ -1006,7 +1006,7 @@ class WindowsToolchainTest(BaseToolchainTest): def test_clang(self): # We'll pick clang if nothing else is found. paths = { - k: v for k, v in self.PATHS.iteritems() + k: v for k, v in self.PATHS.items() if os.path.basename(k) not in ('cl', 'clang-cl', 'gcc') } self.do_toolchain_test(paths, { diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py index 8ec33a8b7..ba046ed12 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import copy import re @@ -10,7 +10,7 @@ import types import unittest from fnmatch import fnmatch -from StringIO import StringIO +from io import StringIO from textwrap import dedent from mozunit import ( @@ -43,7 +43,7 @@ class CompilerPreprocessor(Preprocessor): # Hack around it enough that the configure tests work properly. context = self.context def normalize_numbers(value): - if isinstance(value, types.StringTypes): + if isinstance(value, (str,)): if value[-1:] == 'L' and value[:-1].isdigit(): value = int(value[:-1]) return value @@ -53,7 +53,7 @@ class CompilerPreprocessor(Preprocessor): return self.HAS_FEATURE.sub(r'\1\2', expr) self.context = self.Context( (normalize_has_feature(k), normalize_numbers(v)) - for k, v in context.iteritems() + for k, v in context.items() ) try: return Preprocessor.do_if(self, normalize_has_feature(expression), @@ -95,7 +95,7 @@ class TestCompilerPreprocessor(unittest.TestCase): input.name = 'foo' pp.do_include(input) - self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"') + self.assertEqual(pp.out.getvalue(), '1 . 2 . c "D"') def test_condition(self): pp = CompilerPreprocessor({ @@ -125,7 +125,7 @@ class TestCompilerPreprocessor(unittest.TestCase): input.name = 'foo' pp.do_include(input) - self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue()) + self.assertEqual('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue()) class FakeCompiler(dict): @@ -164,9 +164,9 @@ class FakeCompiler(dict): ''' def __init__(self, *definitions): for definition in definitions: - if all(not isinstance(d, dict) for d in definition.itervalues()): + if all(not isinstance(d, dict) for d in definition.values()): definition = {None: definition} - for key, value in definition.iteritems(): + for key, value in definition.items(): self.setdefault(key, {}).update(value) def __call__(self, stdin, args): @@ -178,14 +178,14 @@ class FakeCompiler(dict): pp = CompilerPreprocessor(self[None]) def apply_defn(defn): - for k, v in defn.iteritems(): + for k, v in defn.items(): if v is False: if k in pp.context: del pp.context[k] else: pp.context[k] = v - for glob, defn in self.iteritems(): + for glob, defn in self.items(): if glob and not glob.startswith('-') and fnmatch(file, glob): apply_defn(defn) @@ -216,7 +216,7 @@ class TestFakeCompiler(unittest.TestCase): 'A': '1', 'B': '2', }) - self.assertEquals(compiler(None, ['-E', 'file']), + self.assertEqual(compiler(None, ['-E', 'file']), (0, '1 2 C', '')) compiler = FakeCompiler({ @@ -238,25 +238,25 @@ class TestFakeCompiler(unittest.TestCase): 'B': '42', }, }) - self.assertEquals(compiler(None, ['-E', 'file']), + self.assertEqual(compiler(None, ['-E', 'file']), (0, '1 2 C', '')) - self.assertEquals(compiler(None, ['-E', '-foo', 'file']), + self.assertEqual(compiler(None, ['-E', '-foo', 'file']), (0, '1 2 foo', '')) - self.assertEquals(compiler(None, ['-E', '-bar', 'file']), + self.assertEqual(compiler(None, ['-E', '-bar', 'file']), (0, '1 bar bar', '')) - self.assertEquals(compiler(None, ['-E', '-qux', 'file']), + self.assertEqual(compiler(None, ['-E', '-qux', 'file']), (0, '1 B C', '')) - self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']), + self.assertEqual(compiler(None, ['-E', '-foo', '-bar', 'file']), (0, '1 bar bar', '')) - self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']), + self.assertEqual(compiler(None, ['-E', '-bar', '-foo', 'file']), (0, '1 bar foo', '')) - self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']), + self.assertEqual(compiler(None, ['-E', '-bar', '-qux', 'file']), (0, '1 B bar', '')) - self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']), + self.assertEqual(compiler(None, ['-E', '-qux', '-bar', 'file']), (0, '1 bar bar', '')) - self.assertEquals(compiler(None, ['-E', 'file.c']), + self.assertEqual(compiler(None, ['-E', 'file.c']), (0, '1 42 C', '')) - self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']), + self.assertEqual(compiler(None, ['-E', '-bar', 'file.c']), (0, '1 bar bar', '')) def test_multiple_definitions(self): @@ -267,7 +267,7 @@ class TestFakeCompiler(unittest.TestCase): 'C': 3, }) - self.assertEquals(compiler, { + self.assertEqual(compiler, { None: { 'A': 1, 'B': 2, @@ -282,7 +282,7 @@ class TestFakeCompiler(unittest.TestCase): 'C': 3, }) - self.assertEquals(compiler, { + self.assertEqual(compiler, { None: { 'A': 1, 'B': 4, @@ -302,7 +302,7 @@ class TestFakeCompiler(unittest.TestCase): }, }) - self.assertEquals(compiler, { + self.assertEqual(compiler, { None: { 'A': 1, 'B': 4, @@ -330,7 +330,7 @@ class TestFakeCompiler(unittest.TestCase): }, }) - self.assertEquals(compiler, { + self.assertEqual(compiler, { None: { 'A': 1, 'B': 2, @@ -370,7 +370,7 @@ class CompilerResult(ReadOnlyNamespace): def __add__(self, other): assert isinstance(other, dict) result = copy.deepcopy(self.__dict__) - for k, v in other.iteritems(): + for k, v in other.items(): if k == 'flags': result.setdefault(k, []).extend(v) else: @@ -381,7 +381,7 @@ class CompilerResult(ReadOnlyNamespace): class TestCompilerResult(unittest.TestCase): def test_compiler_result(self): result = CompilerResult() - self.assertEquals(result.__dict__, { + self.assertEqual(result.__dict__, { 'wrapper': [], 'compiler': mozpath.abspath(''), 'version': '', @@ -397,7 +397,7 @@ class TestCompilerResult(unittest.TestCase): language='C', flags=['-std=gnu99'], ) - self.assertEquals(result.__dict__, { + self.assertEqual(result.__dict__, { 'wrapper': [], 'compiler': mozpath.abspath('/usr/bin/gcc'), 'version': '4.2.1', @@ -407,7 +407,7 @@ class TestCompilerResult(unittest.TestCase): }) result2 = result + {'flags': ['-m32']} - self.assertEquals(result2.__dict__, { + self.assertEqual(result2.__dict__, { 'wrapper': [], 'compiler': mozpath.abspath('/usr/bin/gcc'), 'version': '4.2.1', @@ -416,14 +416,14 @@ class TestCompilerResult(unittest.TestCase): 'flags': ['-std=gnu99', '-m32'], }) # Original flags are untouched. - self.assertEquals(result.flags, ['-std=gnu99']) + self.assertEqual(result.flags, ['-std=gnu99']) result3 = result + { 'compiler': '/usr/bin/gcc-4.7', 'version': '4.7.3', 'flags': ['-m32'], } - self.assertEquals(result3.__dict__, { + self.assertEqual(result3.__dict__, { 'wrapper': [], 'compiler': mozpath.abspath('/usr/bin/gcc-4.7'), 'version': '4.7.3', diff --git a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py index ac35d745f..cdd8ece1b 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import os diff --git a/python/mozbuild/mozbuild/test/configure/test_util.py b/python/mozbuild/mozbuild/test/configure/test_util.py index 9f9575fd0..c2db1a628 100644 --- a/python/mozbuild/mozbuild/test/configure/test_util.py +++ b/python/mozbuild/mozbuild/test/configure/test_util.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals + import logging import os @@ -11,7 +11,7 @@ import textwrap import unittest import sys -from StringIO import StringIO +from io import StringIO from mozunit import main from mozpack import path as mozpath @@ -434,11 +434,11 @@ class TestLogSubprocessOutput(unittest.TestCase): except SystemExit as e: status = e.code - self.assertEquals(status, 0) + self.assertEqual(status, 0) quote_char = "'" if getpreferredencoding().lower() == 'utf-8': quote_char = '\u00B4'.encode('utf-8') - self.assertEquals(out.getvalue().strip(), quote_char) + self.assertEqual(out.getvalue().strip(), quote_char) class TestVersion(unittest.TestCase): diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py index 3229c3f77..82d250fab 100644 --- a/python/mozbuild/mozbuild/testing.py +++ b/python/mozbuild/mozbuild/testing.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals -import cPickle as pickle + +import pickle as pickle import os import sys @@ -70,7 +70,7 @@ REFTEST_FLAVORS = ('crashtest', 'reftest') WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',) def all_test_flavors(): - return ([v[0] for v in TEST_MANIFESTS.values()] + + return ([v[0] for v in list(TEST_MANIFESTS.values())] + list(REFTEST_FLAVORS) + list(WEB_PLATFORM_TESTS_FLAVORS)) @@ -210,7 +210,7 @@ def install_test_files(topsrcdir, topobjdir, tests_root, test_objs): only a few tests need to be run. """ flavor_info = {flavor: (root, prefix, install) - for (flavor, root, prefix, install) in TEST_MANIFESTS.values()} + for (flavor, root, prefix, install) in list(TEST_MANIFESTS.values())} objdir_dest = mozpath.join(topobjdir, tests_root) converter = SupportFilesConverter() @@ -292,7 +292,7 @@ def read_wpt_manifest(context, paths): paths_file = os.path.join(context.config.topsrcdir, "testing", "web-platform", "tests", "tools", "localpaths.py") _globals = {"__file__": paths_file} - execfile(paths_file, _globals) + exec(compile(open(paths_file, "rb").read(), paths_file, 'exec'), _globals) import manifest as wptmanifest finally: sys.path = old_path diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index 4ea227dc0..79cb15713 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -5,7 +5,7 @@ # This file contains miscellaneous utility functions that don't belong anywhere # in particular. -from __future__ import absolute_import, unicode_literals, print_function + import argparse import collections @@ -21,6 +21,9 @@ import stat import sys import time import types +import pprint +import six +import subprocess from collections import ( defaultdict, @@ -36,7 +39,7 @@ from io import ( if sys.version_info[0] == 3: str_type = str else: - str_type = basestring + str_type = str if sys.platform == 'win32': _kernel32 = ctypes.windll.kernel32 @@ -78,7 +81,7 @@ def hash_file(path, hasher=None): return h.hexdigest() -class EmptyValue(unicode): +class EmptyValue(str): """A dummy type that behaves like an empty string and sequence. This type exists in order to support @@ -92,7 +95,7 @@ class EmptyValue(unicode): class ReadOnlyNamespace(object): """A class for objects with immutable attributes set at initialization.""" def __init__(self, **kwargs): - for k, v in kwargs.iteritems(): + for k, v in kwargs.items(): super(ReadOnlyNamespace, self).__setattr__(k, v) def __delattr__(self, key): @@ -224,7 +227,7 @@ class FileAvoidWrite(BytesIO): self.mode = mode def write(self, buf): - if isinstance(buf, unicode): + if isinstance(buf, str): buf = buf.encode('utf-8') BytesIO.write(self, buf) @@ -267,6 +270,10 @@ class FileAvoidWrite(BytesIO): if 'b' in self.mode: writemode += 'b' with open(self.name, writemode) as file: + if 'b' in self.mode and isinstance(buf, str): + buf = buf.encode('utf-8') + elif 'b' not in self.mode and isinstance(buf, bytes): + buf = buf.decode('utf-8') file.write(buf) if self._capture_diff: @@ -381,7 +388,7 @@ class ListMixin(object): def __add__(self, other): # Allow None and EmptyValue is a special case because it makes undefined # variable references in moz.build behave better. - other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other + other = [] if isinstance(other, (type(None), EmptyValue)) else other if not isinstance(other, list): raise ValueError('Only lists can be appended to lists.') @@ -390,7 +397,7 @@ class ListMixin(object): return new_list def __iadd__(self, other): - other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other + other = [] if isinstance(other, (type(None), EmptyValue)) else other if not isinstance(other, list): raise ValueError('Only lists can be appended to lists.') @@ -561,14 +568,14 @@ def FlagsFactory(flags): functions below. """ assert isinstance(flags, dict) - assert all(isinstance(v, type) for v in flags.values()) + assert all(isinstance(v, type) for v in list(flags.values())) class Flags(object): - __slots__ = flags.keys() + __slots__ = list(flags.keys()) _flags = flags def update(self, **kwargs): - for k, v in kwargs.iteritems(): + for k, v in kwargs.items(): setattr(self, k, v) def __getattr__(self, name): @@ -1006,8 +1013,6 @@ def TypedNamedTuple(name, fields): 'got %s, expected %s' % (fname, type(value), ftype)) - super(TypedTuple, self).__init__(*args, **kwargs) - TypedTuple._fields = fields return TypedTuple @@ -1099,14 +1104,14 @@ def group_unified_files(files, unified_prefix, unified_suffix, # issue. So we do a little dance to filter it out ourselves. dummy_fill_value = ("dummy",) def filter_out_dummy(iterable): - return itertools.ifilter(lambda x: x != dummy_fill_value, + return filter(lambda x: x != dummy_fill_value, iterable) # From the itertools documentation, slightly modified: def grouper(n, iterable): "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" args = [iter(iterable)] * n - return itertools.izip_longest(fillvalue=dummy_fill_value, *args) + return itertools.zip_longest(fillvalue=dummy_fill_value, *args) for i, unified_group in enumerate(grouper(files_per_unified_file, files)): @@ -1123,7 +1128,7 @@ def pair(iterable): [(1,2), (3,4), (5,6)] ''' i = iter(iterable) - return itertools.izip_longest(i, i) + return itertools.zip_longest(i, i) VARIABLES_RE = re.compile('\$\((\w+)\)') @@ -1141,7 +1146,7 @@ def expand_variables(s, variables): value = variables.get(name) if not value: continue - if not isinstance(value, types.StringTypes): + if not isinstance(value, (str,)): value = ' '.join(value) result += value return result @@ -1168,7 +1173,7 @@ class EnumStringComparisonError(Exception): pass -class EnumString(unicode): +class EnumString(str): '''A string type that only can have a limited set of values, similarly to an Enum, and can only be compared against that set of values. @@ -1185,8 +1190,8 @@ class EnumString(unicode): def __eq__(self, other): if other not in self.POSSIBLE_VALUES: raise EnumStringComparisonError( - 'Can only compare with %s' - % ', '.join("'%s'" % v for v in self.POSSIBLE_VALUES)) + '%s is not in %s' + % (other, ', '.join("'%s'" % v for v in self.POSSIBLE_VALUES))) return super(EnumString, self).__eq__(other) def __ne__(self, other): @@ -1204,14 +1209,14 @@ def _escape_char(c): # quoting could be done with either ' or ". if c == "'": return "\\'" - return unicode(c.encode('unicode_escape')) + return str(c.encode('unicode_escape')) # Mapping table between raw characters below \x80 and their escaped # counterpart, when they differ _INDENTED_REPR_TABLE = { c: e for c, e in map(lambda x: (x, _escape_char(x)), - map(unichr, range(128))) + map(chr, range(128))) if c != e } # Regexp matching all characters to escape. @@ -1219,7 +1224,33 @@ _INDENTED_REPR_RE = re.compile( '([' + ''.join(_INDENTED_REPR_TABLE.values()) + ']+)') +# Stolen from gecko master [1] +# [1] https://github.com/mozilla/gecko-dev/blob/4165a2e843c494bfb3e35d8a1fbf9c61209e3675/python/mozbuild/mozbuild/util.py#L1286 + +# The default PrettyPrinter has some issues with UTF-8, so we need to override +# some stuff here. +class _PrettyPrinter(pprint.PrettyPrinter): + def format(self, object, context, maxlevels, level): + if not (isinstance(object, six.text_type) or + isinstance(object, six.binary_type)): + return super(_PrettyPrinter, self).format( + object, context, maxlevels, level) + # This is super hacky and weird, but the output of 'repr' actually + # varies based on the default I/O encoding of the process, which isn't + # necessarily utf-8. Instead we open a new shell and ask what the repr + # WOULD be assuming the default encoding is utf-8. If you can come up + # with a better way of doing this without simply re-implementing the + # logic of "repr", please replace this. + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'utf-8' + ret = six.ensure_text(subprocess.check_output( + [sys.executable], input='print(repr(%s))' % repr(object), + universal_newlines=True, env=env, encoding='utf-8')).strip() + return (ret, True, False) + def indented_repr(o, indent=4): + return _PrettyPrinter(indent=indent).pformat(o) + '''Similar to repr(), but returns an indented representation of the object One notable difference with repr is that the returned representation @@ -1242,7 +1273,7 @@ def indented_repr(o, indent=4): elif isinstance(o, bytes): yield 'b' yield repr(o) - elif isinstance(o, unicode): + elif isinstance(o, str): yield "'" # We want a readable string (non escaped unicode), but some # special characters need escaping (e.g. \n, \t, etc.) @@ -1272,11 +1303,11 @@ def encode(obj, encoding='utf-8'): if isinstance(obj, dict): return { encode(k, encoding): encode(v, encoding) - for k, v in obj.iteritems() + for k, v in obj.items() } if isinstance(obj, bytes): return obj - if isinstance(obj, unicode): + if isinstance(obj, str): return obj.encode(encoding) if isinstance(obj, Iterable): return [encode(i, encoding) for i in obj] diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index 38d06d71d..c67c046f3 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -531,9 +531,9 @@ def verify_python_version(log_handle): our = LooseVersion('%d.%d.%d' % (major, minor, micro)) - if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION: - log_handle.write('Python %s or greater (but not Python 3) is ' - 'required to build. ' % MINIMUM_PYTHON_VERSION) + if our < MINIMUM_PYTHON_VERSION: + log_handle.write('Python %s or greater is required to build. ' + % MINIMUM_PYTHON_VERSION) log_handle.write('You are running Python %s.\n' % our) if os.name in ('nt', 'ce'): diff --git a/python/mozbuild/mozpack/chrome/manifest.py b/python/mozbuild/mozpack/chrome/manifest.py index c91b99cf1..02eac0dbb 100644 --- a/python/mozbuild/mozpack/chrome/manifest.py +++ b/python/mozbuild/mozpack/chrome/manifest.py @@ -2,11 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + import re import os -from urlparse import urlparse +from urllib.parse import urlparse import mozpack.path as mozpath from mozpack.chrome.flags import Flags from mozpack.errors import errors @@ -316,7 +316,7 @@ class ManifestContract(ManifestEntry): return self.serialize(self.contractID, self.cid) # All manifest classes by their type name. -MANIFESTS_TYPES = dict([(c.type, c) for c in globals().values() +MANIFESTS_TYPES = dict([(c.type, c) for c in list(globals().values()) if type(c) == type and issubclass(c, ManifestEntry) and hasattr(c, 'type') and c.type]) diff --git a/python/mozbuild/mozpack/copier.py b/python/mozbuild/mozpack/copier.py index 1e521e52b..43ed9be4a 100644 --- a/python/mozbuild/mozpack/copier.py +++ b/python/mozbuild/mozpack/copier.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + import os import stat @@ -113,7 +113,7 @@ class FileRegistry(object): ''' Return all paths stored in the container, in the order they were added. ''' - return self._files.keys() + return list(self._files.keys()) def __len__(self): ''' @@ -146,7 +146,7 @@ class FileRegistry(object): for path, file in registry: (...) ''' - return self._files.iteritems() + return iter(self._files.items()) def required_directories(self): ''' @@ -155,7 +155,7 @@ class FileRegistry(object): unspecified (virtual) root directory (and do not include said root directory). ''' - return set(k for k, v in self._required_directories.items() if v > 0) + return set(k for k, v in list(self._required_directories.items()) if v > 0) def output_to_inputs_tree(self): ''' @@ -295,7 +295,7 @@ class FileCopier(FileRegistry): Returns a FileCopyResult that details what changed. ''' - assert isinstance(destination, basestring) + assert isinstance(destination, str) assert not os.path.exists(destination) or os.path.isdir(destination) result = FileCopyResult() @@ -563,7 +563,7 @@ class Jarrer(FileRegistry, BaseFile): def exists(self): return self.deflater is not None - if isinstance(dest, basestring): + if isinstance(dest, str): dest = Dest(dest) assert isinstance(dest, Dest) diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index 8ce353375..bf35e39b8 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + import errno import os @@ -57,7 +57,7 @@ else: def _copyfile(src, dest): # False indicates `dest` should be overwritten if it exists already. - if isinstance(src, unicode) and isinstance(dest, unicode): + if isinstance(src, str) and isinstance(dest, str): _CopyFileW(src, dest, False) elif isinstance(src, str) and isinstance(dest, str): _CopyFileA(src, dest, False) @@ -164,7 +164,7 @@ class BaseFile(object): disabled when skip_if_older is False. Returns whether a copy was actually performed (True) or not (False). ''' - if isinstance(dest, basestring): + if isinstance(dest, str): dest = Dest(dest) else: assert isinstance(dest, Dest) @@ -278,11 +278,11 @@ class ExecutableFile(File): ''' def copy(self, dest, skip_if_older=True): real_dest = dest - if not isinstance(dest, basestring): + if not isinstance(dest, str): fd, dest = mkstemp() os.close(fd) os.remove(dest) - assert isinstance(dest, basestring) + assert isinstance(dest, str) # If File.copy didn't actually copy because dest is newer, check the # file sizes. If dest is smaller, it means it is already stripped and # elfhacked, so we can skip. @@ -319,7 +319,7 @@ class AbsoluteSymlinkFile(File): File.__init__(self, path) def copy(self, dest, skip_if_older=True): - assert isinstance(dest, basestring) + assert isinstance(dest, str) # The logic in this function is complicated by the fact that symlinks # aren't universally supported. So, where symlinks aren't supported, we @@ -410,7 +410,7 @@ class HardlinkFile(File): ''' def copy(self, dest, skip_if_older=True): - assert isinstance(dest, basestring) + assert isinstance(dest, str) if not hasattr(os, 'link'): return super(HardlinkFile, self).copy( @@ -471,7 +471,7 @@ class ExistingFile(BaseFile): self.required = required def copy(self, dest, skip_if_older=True): - if isinstance(dest, basestring): + if isinstance(dest, str): dest = Dest(dest) else: assert isinstance(dest, Dest) @@ -517,7 +517,7 @@ class PreprocessedFile(BaseFile): ''' Invokes the preprocessor to create the destination file. ''' - if isinstance(dest, basestring): + if isinstance(dest, str): dest = Dest(dest) else: assert isinstance(dest, Dest) @@ -657,7 +657,7 @@ class XPTFile(GeneratedFile): the individual XPTs to link. skip_if_older is ignored. ''' - if isinstance(dest, basestring): + if isinstance(dest, str): dest = Dest(dest) assert isinstance(dest, Dest) @@ -1108,7 +1108,7 @@ class ComposedFinder(BaseFinder): from mozpack.copier import FileRegistry self.files = FileRegistry() - for base, finder in sorted(finders.iteritems()): + for base, finder in sorted(finders.items()): if self.files.contains(base): self.files.remove(base) for p, f in finder.find(''): diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py index 27c66634b..f79b40086 100644 --- a/python/mozbuild/mozpack/manifests.py +++ b/python/mozbuild/mozpack/manifests.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals - from contextlib import contextmanager import json @@ -116,7 +114,7 @@ class InstallManifest(object): self._source_files = set() if path or fileobj: - with _auto_fileobj(path, fileobj, 'rb') as fh: + with _auto_fileobj(path, fileobj, 'r') as fh: self._source_files.add(fh.name) self._load_from_fileobj(fh) @@ -175,7 +173,7 @@ class InstallManifest(object): dest, content = fields[1:] self.add_content( - self._decode_field_entry(content).encode('utf-8'), dest) + self._decode_field_entry(content), dest) continue # Don't fail for non-actionable items, allowing @@ -228,7 +226,7 @@ class InstallManifest(object): It is an error if both are specified. """ - with _auto_fileobj(path, fileobj, 'wb') as fh: + with _auto_fileobj(path, fileobj, 'w') as fh: fh.write('%d\n' % self.CURRENT_VERSION) for dest in sorted(self._dests): @@ -242,13 +240,11 @@ class InstallManifest(object): for path in paths: source = mozpath.join(base, path) parts = ['%d' % type, mozpath.join(dest, path), source] - fh.write('%s\n' % self.FIELD_SEPARATOR.join( - p.encode('utf-8') for p in parts)) + fh.write('%s\n' % self.FIELD_SEPARATOR.join(parts)) else: parts = ['%d' % entry[0], dest] parts.extend(entry[1:]) - fh.write('%s\n' % self.FIELD_SEPARATOR.join( - p.encode('utf-8') for p in parts)) + fh.write('%s\n' % self.FIELD_SEPARATOR.join(parts)) def add_link(self, source, dest): """Add a link to this manifest. @@ -439,7 +435,7 @@ class InstallManifest(object): if install_type == self.CONTENT: # GeneratedFile expect the buffer interface, which the unicode # type doesn't have, so encode to a str. - content = self._decode_field_entry(entry[1]).encode('utf-8') + content = self._decode_field_entry(entry[1]) registry.add(dest, GeneratedFile(content)) continue diff --git a/python/mozbuild/mozpack/mozjar.py b/python/mozbuild/mozpack/mozjar.py index a723fd2c0..0b04d233c 100644 --- a/python/mozbuild/mozpack/mozjar.py +++ b/python/mozbuild/mozpack/mozjar.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import + from io import BytesIO import struct @@ -14,9 +14,10 @@ from zipfile import ( ZIP_DEFLATED, ) from collections import OrderedDict -from urlparse import urlparse, ParseResult +from urllib.parse import urlparse, ParseResult import mozpack.path as mozpath from mozbuild.util import memoize +from functools import reduce JAR_STORED = ZIP_STORED @@ -72,7 +73,7 @@ class JarStruct(object): an instance with empty fields. ''' assert self.MAGIC and isinstance(self.STRUCT, OrderedDict) - self.size_fields = set(t for t in self.STRUCT.itervalues() + self.size_fields = set(t for t in self.STRUCT.values() if not t in JarStruct.TYPE_MAPPING) self._values = {} if data: @@ -94,7 +95,7 @@ class JarStruct(object): # For all fields used as other fields sizes, keep track of their value # separately. sizes = dict((t, 0) for t in self.size_fields) - for name, t in self.STRUCT.iteritems(): + for name, t in self.STRUCT.items(): if t in JarStruct.TYPE_MAPPING: value, size = JarStruct.get_data(t, data[offset:]) else: @@ -113,7 +114,7 @@ class JarStruct(object): Initialize an instance with empty fields. ''' self.signature = self.MAGIC - for name, t in self.STRUCT.iteritems(): + for name, t in self.STRUCT.items(): if name in self.size_fields: continue self._values[name] = 0 if t in JarStruct.TYPE_MAPPING else '' @@ -138,9 +139,9 @@ class JarStruct(object): from self.STRUCT. ''' serialized = struct.pack('" % (self.__class__.__name__, @@ -374,7 +375,7 @@ class JarReader(object): entries = self.entries if not entries: return JAR_STORED - return max(f['compression'] for f in entries.itervalues()) + return max(f['compression'] for f in entries.values()) @property def entries(self): @@ -390,7 +391,7 @@ class JarReader(object): preload = JarStruct.get_data('uint32', self._data)[0] entries = OrderedDict() offset = self._cdir_end['cdir_offset'] - for e in xrange(self._cdir_end['cdir_entries']): + for e in range(self._cdir_end['cdir_entries']): entry = JarCdirEntry(self._data[offset:]) offset += entry.size # Creator host system. 0 is MSDOS, 3 is Unix @@ -452,7 +453,7 @@ class JarReader(object): for file in jarReader: ... ''' - for entry in self.entries.itervalues(): + for entry in self.entries.values(): yield self._getreader(entry) def __getitem__(self, name): @@ -547,7 +548,7 @@ class JarWriter(object): headers = {} preload_size = 0 # Prepare central directory entries - for entry, content in self._contents.itervalues(): + for entry, content in self._contents.values(): header = JarLocalFileHeader() for name in entry.STRUCT: if name in header: @@ -562,7 +563,7 @@ class JarWriter(object): end['disk_entries'] = len(self._contents) end['cdir_entries'] = end['disk_entries'] end['cdir_size'] = reduce(lambda x, y: x + y[0].size, - self._contents.values(), 0) + list(self._contents.values()), 0) # On optimized archives, store the preloaded size and the central # directory entries, followed by the first end of central directory. if self._optimize: @@ -571,18 +572,18 @@ class JarWriter(object): if preload_size: preload_size += offset self._data.write(struct.pack('