From 073c1d1c31f8262c8f1ae04006796215d2fdab1a Mon Sep 17 00:00:00 2001 From: Favonia Date: Sun, 18 Jan 2015 11:00:47 -0500 Subject: [PATCH] feat(python): add python 3 compatibility Add Python 3 compatibility through python-six library. Closes #393. --- .travis.yml | 1 + bin/leantags | 11 ++-- bin/linja | 123 ++++++++++++++++++----------------- script/demangle_cpptype.py | 8 ++- script/dropbox_upload.py | 17 +++-- src/cmake/Modules/cpplint.py | 12 ++-- 6 files changed, 96 insertions(+), 76 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67855684e..59cca4c15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -189,6 +189,7 @@ install: - if [[ $REPO == BLESSED && ($UPLOAD || $BUILD_DOXYGEN == TRUE) ]]; then sudo apt-get -qq install python python-pip; sudo pip install dropbox; + sudo pip install six; fi - if [[ $TESTCOV == ON ]]; then wget http://downloads.sourceforge.net/ltp/lcov-1.10.tar.gz; diff --git a/bin/leantags b/bin/leantags index 4f216ded2..fc43699c2 100755 --- a/bin/leantags +++ b/bin/leantags @@ -5,6 +5,9 @@ # # Author: Soonho Kong +# Python 2/3 compatibility +from __future__ import print_function + import argparse import fnmatch import os @@ -63,7 +66,7 @@ def build_line_to_byteoffset_map(ilean_filename): return (map, lines) def extract_filename_from_info(info): - return next((item['filename'] for item in info if 'filename' in item.keys()), None) + return next((item['filename'] for item in info if 'filename' in item), None) def convert_position_to_etag_style(info): """Convert the position format from (line, col) to (line, byteoffset)""" @@ -131,12 +134,12 @@ def get_etag_def(info): return "" def print_item(item): - print "\t%s\t%-60s:%4d:%4d:%6d - %s" \ - % (item['category'], item['filename'], item['linenum'], item['col'], item['offset'], item['declname']) + print("\t%s\t%-60s:%4d:%4d:%6d - %s" \ + % (item['category'], item['filename'], item['linenum'], item['col'], item['offset'], item['declname'])) def print_items(items): for item in items: - print get_etag_def_entry(item), + print(get_etag_def_entry(item), end=' ') def find_makefile(path, makefile_names): """ Find makefile in a given directory. diff --git a/bin/linja b/bin/linja index c72c84a66..2409b8721 100755 --- a/bin/linja +++ b/bin/linja @@ -6,6 +6,12 @@ # # Author: Soonho Kong # + +# Python 2/3 compatibility +from __future__ import print_function +import six +from six.moves import filter, map + import argparse import fnmatch import glob @@ -19,11 +25,12 @@ import subprocess import sys import tempfile import threading -import urllib +from six.moves import urllib -# Enforce subprocesses to use 'utf-8' -reload(sys) -sys.setdefaultencoding("utf-8") +# Enforce subprocesses to use 'utf-8' in Python 2 +if six.PY2: + reload(sys) + sys.setdefaultencoding("utf-8") # Fixate the path separator as '\' on Windows Platform # even if users are on CYGWIN/MSYS2 environment @@ -53,6 +60,7 @@ g_flycheck_footer = "FLYCHECK_END" g_logger = logging.getLogger('linja_logger') g_debug_mode = False +@six.python_2_unicode_compatible class FlycheckItem: def __init__(self, filename, lineno, colno, ty, msg): self.filename = filename @@ -62,8 +70,6 @@ class FlycheckItem: self.msg = msg pass def __str__(self): - return unicode(self).encode('utf-8') - def __unicode__(self): ret = g_flycheck_header + " " + self.ty.upper() + "\n" ret += "%s:%d:%d: %s: %s" % (self.filename, self.lineno, self.colno, self.ty, self.msg) + "\n" ret += g_flycheck_footer @@ -87,13 +93,12 @@ class FlycheckItem: msg = msg.strip() return cls(filename, lineno, colno, ty, msg) +@six.python_2_unicode_compatible class FlycheckItemList: def __init__(self, items): self.items = items def __str__(self): - return unicode(self).encode('utf-8') - def __unicode__(self): - return "\n".join([unicode(item) for item in self.items]) + return "\n".join([six.text_type(item) for item in self.items]) def __getitem__(self, i): return self.items[i] def __len__(self): @@ -101,7 +106,7 @@ class FlycheckItemList: def sort(self): self.items = sorted(self.items) def filter(self, pred): - self.items = filter(pred, self.items) + self.items = list(filter(pred, self.items)) def append(self, item): self.items.append(item) def truncate(self, n): @@ -147,10 +152,10 @@ def init_logger(): g_logger.addHandler(fileHandler) def log(msg): - print >> sys.stderr, msg + print(msg, file=sys.stderr) def log_nonewline(msg): - print >> sys.stderr, ("\r%s" % msg), + print(("\r%s" % msg), end=' ', file=sys.stderr) sys.stderr.flush() def error(msg): @@ -226,7 +231,7 @@ def download_ninja_and_save_at(ninja_path): else: url = get_ninja_url() log("Downloading ninja: %s ===> %s\n" % (url, ninja_path)) - urllib.urlretrieve(url, ninja_path, show_download_progress) + urllib.request.urlretrieve(url, ninja_path, show_download_progress) log("\n") if not os.path.isfile(ninja_path): error("failed to download ninja executable from %s" % url) @@ -308,7 +313,7 @@ def check_requirements(): def process_args(args): if (args.flycheck == False and args.flycheck_max_messages != None): error("Please use --flycheck option with --flycheck-max-messages option.") - args.targets = map(process_target, args.targets) + args.targets = list(map(process_target, args.targets)) if args.directory: os.chdir(args.directory) args.project_dir = find_file_upward(g_project_filename) @@ -369,17 +374,17 @@ def parse_arg(argv): return args def debug_status(args): - print "Working Directory =", os.getcwd() - print "" - for key, val in vars(args).iteritems(): - print "Option[" + key + "] =", val - print "" - print "linja path =", g_linja_path - print "lean/bin dir =", g_lean_bin_dir - print "phony targets =", g_phony_targets - print "lean path =", g_lean_path - print "leantags path =", g_leantags_path - print "ninja path =", g_ninja_path + print("Working Directory =", os.getcwd()) + print("") + for key, val in six.iteritems(vars(args)): + print("Option[" + key + "] =", val) + print("") + print("linja path =", g_linja_path) + print("lean/bin dir =", g_lean_bin_dir) + print("phony targets =", g_phony_targets) + print("lean path =", g_lean_path) + print("leantags path =", g_leantags_path) + print("ninja path =", g_ninja_path) def handle_flycheck_failure(out, err, args): if len(args.targets) == 0: @@ -393,13 +398,13 @@ def handle_flycheck_failure(out, err, args): if lean in line and lean != target: failed.add(lean) for failed_file in failed: - print g_flycheck_header, "ERROR" - print "%s:1:0: error: failed to compile %s" % (target, failed_file) - print g_flycheck_footer + print(g_flycheck_header, "ERROR") + print("%s:1:0: error: failed to compile %s" % (target, failed_file)) + print(g_flycheck_footer) if err: - print g_flycheck_header, "ERROR" - print "%s:1:0: error: %s" % (target, err.strip()) - print g_flycheck_footer + print(g_flycheck_header, "ERROR") + print("%s:1:0: error: %s" % (target, err.strip())) + print(g_flycheck_footer) if failed: call_lean(target, args) @@ -411,7 +416,7 @@ def process_lean_output(target, out, args, using_hlean): else: target = target[:-5] + "lean" if (not target.endswith(".lean") and not using_hlean) or (not target.endswith(".hlean") and using_hlean): - print out + print(out) return # Parse, filter, and remove extra items flycheckItemList = FlycheckItemList.fromString(out) @@ -424,7 +429,7 @@ def process_lean_output(target, out, args, using_hlean): flycheckItemList.truncate(n) tooManyItemsError = FlycheckItem(target, 1, 0, "error", "For performance, we only display %d errors/warnings out of %d. (lean-flycheck-max-messages-to-display)" % (n, count)) flycheckItemList.append(tooManyItemsError) - print flycheckItemList + print(flycheckItemList) def call_ninja(args): targets = [] @@ -449,7 +454,7 @@ def call_ninja(args): process_lean_output(targets[0], out, args, args.targets[0].endswith(".hlean")) handle_flycheck_failure(out, err, args) else: - print out + err + print(out + err) return proc.returncode @@ -552,7 +557,7 @@ def clear_cache(args): sys.stderr.write("\n") def build_olean(lean, olean, clean, dlean, ilean, base): - (lean, olean, clean, dlean, ilean, base) = map(escape_ninja_char, (lean, olean, clean, dlean, ilean, base)) + (lean, olean, clean, dlean, ilean, base) = list(map(escape_ninja_char, (lean, olean, clean, dlean, ilean, base))) if clean.startswith(base): str = """build %s %s %s: LEAN %s | %s\n""" % (olean, ilean, clean, lean, dlean) else: @@ -567,52 +572,52 @@ def make_build_ninja(args): with open(os.path.join(args.project_dir, "build.ninja"), "w") as f: lean_files = find_lean_files(args) - print >> f, """rule CLEAN""" - print >> f, """ command = """, - print >> f, """"%s" -t clean""" % g_ninja_path - print >> f, """ description = Cleaning all built files...""" + print("""rule CLEAN""", file=f) + print(""" command = """, end=' ', file=f) + print(""""%s" -t clean""" % g_ninja_path, file=f) + print(""" description = Cleaning all built files...""", file=f) - print >> f, """rule LEAN""" - print >> f, """ depfile = ${DLEAN_FILE}""" - print >> f, """ command = "%s" %s "$in" -o "${OLEAN_FILE}" -c "${CLEAN_FILE}" -i "${ILEAN_FILE}" """ \ - % (g_lean_path, " ".join(args.lean_options)) + print("""rule LEAN""", file=f) + print(""" depfile = ${DLEAN_FILE}""", file=f) + print(""" command = "%s" %s "$in" -o "${OLEAN_FILE}" -c "${CLEAN_FILE}" -i "${ILEAN_FILE}" """ \ + % (g_lean_path, " ".join(args.lean_options)), file=f) - print >> f, """rule LEANTAGS""" - print >> f, """ command = """, + print("""rule LEANTAGS""", file=f) + print(""" command = """, end=' ', file=f) if platform.system() == "Windows": - print >> f, "python ", - print >> f, """"%s" --relative -- $in """ % (g_leantags_path) + print("python ", end=' ', file=f) + print(""""%s" --relative -- $in """ % (g_leantags_path), file=f) - print >> f, "build all: phony", + print("build all: phony", end=' ', file=f) for item in lean_files: - print >> f, " " + escape_ninja_char(item['olean']), - print >> f, "" + print(" " + escape_ninja_char(item['olean']), end=' ', file=f) + print("", file=f) tags_file = "TAGS" - print >> f, "build tags: phony " + tags_file - print >> f, "build " + tags_file + ": LEANTAGS", + print("build tags: phony " + tags_file, file=f) + print("build " + tags_file + ": LEANTAGS", end=' ', file=f) for item in lean_files: - print >> f, " " + escape_ninja_char(item['ilean']), - print >> f, "" + print(" " + escape_ninja_char(item['ilean']), end=' ', file=f) + print("", file=f) - print >> f, """build clean: CLEAN""" + print("""build clean: CLEAN""", file=f) for item in lean_files: - print >> f, build_olean(item['lean'], item['olean'], item['clean'], item['d'], item['ilean'], item['base']) + print(build_olean(item['lean'], item['olean'], item['clean'], item['d'], item['ilean'], item['base']), file=f) - print >> f, """default all""" + print("""default all""", file=f) def make_deps(lean_file, dlean_file, olean_file): with open(dlean_file, "w") as f: deps = [] proc = subprocess.Popen([g_lean_path, "--deps", lean_file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = proc.communicate()[0] - print >> f, olean_file + ": \\" + print(olean_file + ": \\", file=f) for olean_file in output.strip().splitlines(): if olean_file: deps.append(normalize_drive_name(os.path.abspath(olean_file))) deps_str = " " + (" \\\n ".join(deps)) - print >> f, deps_str + print(deps_str, file=f) def make_deps_all_files(args): lean_files = find_lean_files(args) diff --git a/script/demangle_cpptype.py b/script/demangle_cpptype.py index e809ef8bd..fa67ae703 100755 --- a/script/demangle_cpptype.py +++ b/script/demangle_cpptype.py @@ -16,6 +16,10 @@ # # It calls "c++filt" to do the work. # + +# Python 2/3 compatibility +from __future__ import print_function + import re import sys import subprocess @@ -29,14 +33,14 @@ cppfilt_option = "--types" def process_line(line): result = pattern.match(line); if result == None: - print line, + print(line, end=' ') else: p = subprocess.Popen(cppfilt + " " + cppfilt_option + " " + result.group(2), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ty = p.stdout.readlines()[0].strip() retval= p.wait() new_str = re.sub(pattern_str, r"\1" + ty + r"\3", line); - print new_str, + print(new_str, end=' ') if len(sys.argv) > 1: for line in fileinput.input(): diff --git a/script/dropbox_upload.py b/script/dropbox_upload.py index f7f29f00c..ad354e705 100755 --- a/script/dropbox_upload.py +++ b/script/dropbox_upload.py @@ -6,6 +6,9 @@ # Author: Soonho Kong # +# Python 2/3 compatibility +from __future__ import print_function + import dropbox import os import argparse @@ -19,10 +22,10 @@ parser.add_argument('--deletelist', type=str, help='File containing a list of args = parser.parse_args() if not args.srcpath and not args.copylist and not args.deletelist: - print "You need to specify one of the following options:" - print " --srcpath," - print " --copylist," - print " --deletelist" + print("You need to specify one of the following options:") + print(" --srcpath,") + print(" --copylist,") + print(" --deletelist") exit(1) access_token = args.dropbox_token @@ -33,7 +36,7 @@ try: client = dropbox.client.DropboxClient(access_token) client.account_info() except: - print "Failed to connect to Dropbox. Please check the access token." + print("Failed to connect to Dropbox. Please check the access token.") exit(1) count = 0 @@ -86,7 +89,7 @@ if args.copylist: try: copylist_handle = open(copylist, "r") except IOError: - print 'Failed to open ' + copylist + print('Failed to open ' + copylist) for line in copylist_handle: fullpath = os.path.normpath(line.strip()) copy_file_with_retry(fullpath, os.path.normpath(server_path + "/" + fullpath), 5) @@ -98,7 +101,7 @@ if args.deletelist: try: deletelist_handle = open(deletelist, "r") except IOError: - print 'Failed to open ' + deletelist + print('Failed to open ' + deletelist) deletelist_handle = open(deletelist, "r") for line in deletelist_handle: fullpath = os.path.normpath(line.strip()) diff --git a/src/cmake/Modules/cpplint.py b/src/cmake/Modules/cpplint.py index 9cd6c851c..085c4525c 100755 --- a/src/cmake/Modules/cpplint.py +++ b/src/cmake/Modules/cpplint.py @@ -76,6 +76,10 @@ We do a small hack, which is to ignore //'s with "'s after them on the same line, but it is far from perfect (in either direction). """ +# Python 2/3 compatibility +import six +from six.moves import range, xrange + import codecs import copy import getopt @@ -668,7 +672,7 @@ class _CppLintState(object): def PrintErrorCounts(self): """Print a summary of errors by category, and the total.""" - for category, count in self.errors_by_category.iteritems(): + for category, count in six.iteritems(self.errors_by_category): sys.stderr.write('Category \'%s\' errors found: %d\n' % (category, count)) sys.stderr.write('Total errors found: %d\n' % self.error_count) @@ -2837,7 +2841,7 @@ def GetLineWidth(line): The width of the line in column positions, accounting for Unicode combining characters and wide characters. """ - if isinstance(line, unicode): + if isinstance(line, six.string_types): width = 0 for uc in unicodedata.normalize('NFC', line): if unicodedata.east_asian_width(uc) in ('W', 'F'): @@ -3172,7 +3176,7 @@ def _GetTextInside(text, start_pattern): # Give opening punctuations to get the matching close-punctuations. matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(matching_punctuation.itervalues()) + closing_punctuation = set(six.itervalue(matching_punctuation)) # Find the position to start extracting text. match = re.search(start_pattern, text, re.M) @@ -3796,7 +3800,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, # include_state is modified during iteration, so we iterate over a copy of # the keys. - header_keys = include_state.keys() + header_keys = list(include_state): for header in header_keys: (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) fullpath = common_path + header