47c0ae5914
Assume that we have two linja processes running on a system where there
is no ninja installed. Then, the first linja process downloads ninja
from github. If the internet is slow, the second linja process can pick
up the incomplete ninja binary and execute it, which causes an
exception (i.e. "Malformed Mach-o file" error on OSX). An example build
trace is at
https://s3.amazonaws.com/archive.travis-ci.org/jobs/56366771/log.txt
This commit fixes the problem by downloading ninja to a temporary
directory and copy it to "lean/bin/ninja" when it's completed.
748 lines
28 KiB
Python
Executable file
748 lines
28 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (c) 2014 Microsoft Corporation. All rights reserved.
|
|
# Released under Apache 2.0 license as described in the file LICENSE.
|
|
#
|
|
# Author: Soonho Kong
|
|
#
|
|
|
|
# This program contains code snippets from the Python six library
|
|
# released under the following LICENSE:
|
|
#
|
|
# Copyright (c) 2010-2015 Benjamin Peterson
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
# Python 2/3 compatibility
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import fnmatch
|
|
import glob
|
|
import logging
|
|
import logging.handlers
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
|
|
# Python 2/3 compatibility
|
|
if sys.version_info[0] == 2:
|
|
def python_2_unicode_compatible(klass):
|
|
klass.__unicode__ = klass.__str__
|
|
klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
|
|
return klass
|
|
|
|
# Enforce subprocesses to use 'utf-8' in Python 2
|
|
reload(sys)
|
|
sys.setdefaultencoding("utf-8")
|
|
|
|
# Aliases
|
|
text_type = unicode
|
|
import itertools
|
|
filter = itertools.ifilter
|
|
map = itertools.imap
|
|
iteritems = dict.iteritems
|
|
|
|
from urllib import urlretrieve
|
|
elif sys.version_info[0] == 3:
|
|
def python_2_unicode_compatible(klass):
|
|
return klass
|
|
|
|
def my_str(a):
|
|
# The following is a Hack to avoid the following error on Python 3.4 for Windows.
|
|
# UnicodeEncodeError: 'charmap' codec can't encode character '\u2192' in position xxx:
|
|
# character maps to <undefined>
|
|
return str(a).encode('utf-8').decode(sys.stdout.encoding, errors='replace')
|
|
|
|
# Aliases
|
|
text_type = my_str
|
|
iteritems = dict.items
|
|
|
|
from urllib.request import urlretrieve
|
|
else:
|
|
sys.exit('Unsupported Python version')
|
|
|
|
# Fixate the path separator as '\' on Windows Platform
|
|
# even if users are on CYGWIN/MSYS2 environment
|
|
if platform.system() == "Windows":
|
|
os.path.sep = "\\"
|
|
|
|
# Reset MSYSTEM environment variable to enforce native-WINDOWS
|
|
# behavior to subprocesses.
|
|
if "MSYSTEM" in os.environ:
|
|
del os.environ["MSYSTEM"]
|
|
|
|
if platform.system().startswith("MSYS"):
|
|
# In MSYS platform, realpath has a strange behavior.
|
|
# os.path.realpath("c:\a\b\c") => \:\a\b\c
|
|
g_linja_path = os.path.abspath(os.path.normpath(__file__))
|
|
else:
|
|
g_linja_path = os.path.abspath(os.path.realpath(__file__))
|
|
g_lean_bin_dir = os.path.dirname(g_linja_path)
|
|
g_phony_targets = ["all", "clean", "tags", "clear-cache"]
|
|
g_project_filename = ".project"
|
|
g_lean_path = "USE DEFAULT" # System will search automatically
|
|
g_leantags_path = "USE DEFAULT" # System will search automatically
|
|
g_ninja_path = "USE DEFAULT" # System will search automatically
|
|
g_flycheck_header = "FLYCHECK_BEGIN"
|
|
g_flycheck_footer = "FLYCHECK_END"
|
|
g_lean_bin_dep_flag= "@LEAN_BIN_DEP@" == "ON"
|
|
|
|
g_logger = logging.getLogger('linja_logger')
|
|
g_debug_mode = False
|
|
|
|
@python_2_unicode_compatible
|
|
class FlycheckItem:
|
|
def __init__(self, filename, lineno, colno, ty, msg):
|
|
self.filename = filename
|
|
self.lineno = lineno
|
|
self.colno = colno
|
|
self.ty = ty
|
|
self.msg = msg
|
|
pass
|
|
def __str__(self):
|
|
ret = g_flycheck_header + " " + self.ty.upper() + "\n"
|
|
ret += "%s:%d:%d: %s: %s" % (self.filename, self.lineno, self.colno, self.ty, self.msg) + "\n"
|
|
ret += g_flycheck_footer
|
|
return ret
|
|
def __lt__(self, other):
|
|
return (self.filename, self.ty, self.lineno, self.colno) < (other.filename, other.ty, other.lineno, other.colno)
|
|
def loc(self):
|
|
return (self.filename, self.ty, self.lineno, self.colno)
|
|
@classmethod
|
|
def fromString(cls, target, text):
|
|
lines = text.strip().splitlines()
|
|
# Throw the first and last lines (header/footer)
|
|
lines = lines[1:-1]
|
|
try:
|
|
firstLine = lines[0]
|
|
items = [item.strip() for item in firstLine.split(":")]
|
|
filename = items[0]
|
|
lineno = int(items[1])
|
|
colno = int(items[2])
|
|
ty = items[3]
|
|
msg = ":".join(items[4:]) + "\n" + "\n".join(lines[1:])
|
|
msg = msg.strip()
|
|
return cls(filename, lineno, colno, ty, msg)
|
|
except:
|
|
return cls(target, 1, 0, "error", " ".join(lines))
|
|
|
|
@python_2_unicode_compatible
|
|
class FlycheckItemList:
|
|
def __init__(self, items):
|
|
self.items = items
|
|
def __str__(self):
|
|
return "\n".join([text_type(item) for item in self.items])
|
|
def __getitem__(self, i):
|
|
return self.items[i]
|
|
def __len__(self):
|
|
return len(self.items)
|
|
def sort(self):
|
|
self.items = sorted(self.items)
|
|
def filter(self, pred):
|
|
self.items = list(filter(pred, self.items))
|
|
def append(self, item):
|
|
self.items.append(item)
|
|
def truncate(self, n):
|
|
del self.items[n:]
|
|
def removeExtraItemsStartswith(self, text):
|
|
self.sort()
|
|
newItems = self.items[:1]
|
|
i = 1
|
|
while i < len(self.items):
|
|
prev_item = self.items[i-1]
|
|
cur_item = self.items[i]
|
|
if not cur_item.msg.startswith(text) or prev_item.loc() != cur_item.loc():
|
|
newItems.append(cur_item)
|
|
i += 1
|
|
self.items = newItems
|
|
|
|
@classmethod
|
|
def fromString(cls, target, text):
|
|
items = []
|
|
tmpBuffer = ""
|
|
ignore = True
|
|
for line in text.splitlines():
|
|
# I had to add the following line to avoid a crash on Python 3.4 for Windows
|
|
line = line.decode("utf-8")
|
|
if line.startswith(g_flycheck_header):
|
|
tmpBuffer = tmpBuffer + line + "\n"
|
|
ignore = False
|
|
elif line.startswith(g_flycheck_footer):
|
|
tmpBuffer = tmpBuffer + line + "\n"
|
|
items.append(FlycheckItem.fromString(target, tmpBuffer.strip()))
|
|
tmpBuffer = ""
|
|
ignore = True
|
|
elif not ignore:
|
|
tmpBuffer = tmpBuffer + line + "\n"
|
|
return cls(items)
|
|
|
|
def init_logger():
|
|
formatter = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s')
|
|
streamHandler = logging.StreamHandler()
|
|
streamHandler.setFormatter(formatter)
|
|
g_logger.addHandler(streamHandler)
|
|
if g_debug_mode == True:
|
|
fileHandler = logging.FileHandler('./linja.log')
|
|
fileHandler.setFormatter(formatter)
|
|
g_logger.addHandler(fileHandler)
|
|
|
|
def log(msg):
|
|
print(msg, file=sys.stderr)
|
|
|
|
def log_nonewline(msg):
|
|
print(("\r%s" % msg), end=' ', file=sys.stderr)
|
|
sys.stderr.flush()
|
|
|
|
def error(msg):
|
|
log("Error: %s" % msg)
|
|
exit(1)
|
|
|
|
class LinjaException(Exception):
|
|
"""Custom Exception"""
|
|
def __init__(self, msg):
|
|
self.msg = msg
|
|
def __str__(self):
|
|
return self.msg
|
|
|
|
def give_exec_permission(filename):
|
|
"""chmod +x filename"""
|
|
st = os.stat(filename)
|
|
os.chmod(filename, st.st_mode | stat.S_IEXEC)
|
|
|
|
def show_download_progress(*data):
|
|
file_size = int(data[2]/1000)
|
|
total_packets = data[2]/data[1]
|
|
downloaded_packets = data[0]
|
|
log_nonewline("Download: size\t= %i kb, packet: %i/%i" % (file_size, downloaded_packets, total_packets+1))
|
|
|
|
def get_ninja_url():
|
|
prefix = "https://leanprover.github.io/bin/"
|
|
if platform.architecture()[0] == "64bit":
|
|
if platform.system() == "Linux":
|
|
return prefix + "ninja-1.5.1-linux-x86_64"
|
|
elif platform.system() == "Windows":
|
|
return prefix + "ninja-1.5.1-win.exe"
|
|
elif platform.system() == "Darwin":
|
|
return prefix + "ninja-1.5.1-osx"
|
|
if platform.architecture()[0] == "32bit":
|
|
if platform.system() == "Linux":
|
|
return prefix + "ninja-1.5.1-linux-i386"
|
|
elif platform.system() == "Windows":
|
|
pass # TODO(soonhok): add support
|
|
elif platform.system() == "Darwin":
|
|
pass # TODO(soonhok): add support
|
|
error("we do not have ninja executable for this platform: %s" % platform.platform())
|
|
|
|
def build_ninja_and_save_at(ninja_path, platform):
|
|
saved_current_dir = os.getcwd()
|
|
tempdir = tempfile.mkdtemp()
|
|
build_dir = os.path.join(tempdir, "ninja")
|
|
built_ninja_path = os.path.join(build_dir, "ninja")
|
|
cmd_clone_ninja = ["git", "clone", "git://github.com/martine/ninja.git"]
|
|
cmd_checkout_release = ["git", "checkout", "release"]
|
|
cmd_bootstrap = [os.path.join(build_dir, "configure.py"), "--bootstrap", "--platform", platform]
|
|
try:
|
|
os.chdir(tempdir)
|
|
if subprocess.call(cmd_clone_ninja):
|
|
raise LinjaException("Failed to clone ninja repository")
|
|
os.chdir(build_dir)
|
|
if subprocess.call(cmd_checkout_release):
|
|
raise LinjaException("Failed to checkout release branch of ninja")
|
|
if subprocess.call(cmd_bootstrap):
|
|
raise LinjaException("Failed to build ninja")
|
|
shutil.copy2(built_ninja_path, ninja_path)
|
|
except IOError as e:
|
|
error(e)
|
|
except LinjaException as e:
|
|
error(e)
|
|
finally:
|
|
os.chdir(saved_current_dir)
|
|
shutil.rmtree(tempdir)
|
|
return ninja_path
|
|
|
|
def download_ninja_and_save_at(ninja_path):
|
|
if platform.system().startswith("CYGWIN"):
|
|
return build_ninja_and_save_at(ninja_path, "linux")
|
|
else:
|
|
url = get_ninja_url()
|
|
log("Downloading ninja: %s ===> %s\n" % (url, ninja_path))
|
|
tempdir = tempfile.mkdtemp()
|
|
temp_download_path = os.path.join(tempdir, os.path.split(ninja_path)[1])
|
|
urlretrieve(url, temp_download_path, show_download_progress)
|
|
log("\n")
|
|
if not os.path.isfile(temp_download_path):
|
|
error("failed to download ninja executable from %s" % url)
|
|
shutil.copy2(temp_download_path, ninja_path)
|
|
give_exec_permission(ninja_path)
|
|
return ninja_path
|
|
|
|
def find_file_upward(name, path = os.getcwd()):
|
|
project_file = os.path.join(path, name)
|
|
if os.path.isfile(project_file):
|
|
return path
|
|
up = os.path.dirname(path)
|
|
if up != path:
|
|
return find_file_upward(name, up)
|
|
return None
|
|
|
|
def escape_ninja_char(name):
|
|
return name.replace("$", "$$").replace(":", "$:").replace(" ", "$ ")
|
|
|
|
def normalize_drive_name(name):
|
|
if platform.system() == "Windows":
|
|
drive, path = os.path.splitdrive(name)
|
|
if drive == None:
|
|
return name
|
|
else:
|
|
# Leo: return drive.lower() + path
|
|
return path
|
|
else:
|
|
return name
|
|
|
|
def process_target(target):
|
|
if target in g_phony_targets:
|
|
return target
|
|
return normalize_drive_name(os.path.abspath(target))
|
|
|
|
def which(program):
|
|
""" Lookup program in a path """
|
|
import os
|
|
def is_exe(fpath):
|
|
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
|
|
fpath, fname = os.path.split(program)
|
|
if fpath:
|
|
if is_exe(program):
|
|
return program
|
|
else:
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
path = path.strip('"')
|
|
if "sbin" in path:
|
|
continue
|
|
exe_file = os.path.join(path, program)
|
|
if is_exe(exe_file):
|
|
return exe_file
|
|
return None
|
|
|
|
def find_location(exec_name, findLocal=True):
|
|
if findLocal:
|
|
pathname = os.path.join(os.path.dirname(g_linja_path), exec_name)
|
|
if os.path.isfile(pathname):
|
|
return pathname
|
|
pathname = which(exec_name) or os.path.join(g_lean_bin_dir, exec_name)
|
|
pathname = os.path.abspath(pathname)
|
|
return pathname
|
|
|
|
def check_requirements():
|
|
global g_lean_path, g_leantags_path, g_ninja_path
|
|
leantags_exec_name = "leantags"
|
|
lean_exec_name = "lean"
|
|
ninja_exec_name = "ninja"
|
|
if platform.system() == "Windows" or platform.system().startswith("MSYS"):
|
|
lean_exec_name = "lean.exe"
|
|
ninja_exec_name = "ninja.exe"
|
|
|
|
if g_lean_path == "USE DEFAULT":
|
|
g_lean_path = find_location(lean_exec_name, True)
|
|
if g_leantags_path == "USE DEFAULT":
|
|
g_leantags_path = find_location(leantags_exec_name, True)
|
|
if g_ninja_path == "USE DEFAULT":
|
|
g_ninja_path = find_location(ninja_exec_name, False)
|
|
if not os.path.isfile(g_lean_path):
|
|
error("cannot find lean executable at " + os.path.abspath(g_lean_path))
|
|
if not os.path.isfile(g_leantags_path):
|
|
error("cannot find leantags executable at " + os.path.abspath(g_leantags_path))
|
|
if not os.path.isfile(g_ninja_path):
|
|
g_ninja_path = download_ninja_and_save_at(g_ninja_path)
|
|
|
|
def process_args(args):
|
|
if (args.flycheck == False and args.flycheck_max_messages != None):
|
|
error("Please use --flycheck option with --flycheck-max-messages option.")
|
|
args.targets = list(map(process_target, args.targets))
|
|
if args.directory:
|
|
os.chdir(args.directory)
|
|
args.project_dir = find_file_upward(g_project_filename)
|
|
if args.project_dir:
|
|
os.chdir(args.project_dir)
|
|
if args.cache:
|
|
args.cache = process_target(args.cache)
|
|
if len(args.targets) != 1:
|
|
error("--cache option can only be used with one target")
|
|
if not args.cache.endswith(".lean") and not args.cache.endswith(".hlean"):
|
|
error("cache argument has to end with .lean or .hlean")
|
|
if args.cache.endswith(".lean"):
|
|
args.cache = args.cache[:-4] + "clean"
|
|
else:
|
|
args.cache = args.cache[:-5] + "clean"
|
|
args.phony_targets = list(set(g_phony_targets) & set(args.targets))
|
|
if args.verbose:
|
|
g_logger.setLevel(logging.INFO)
|
|
return args
|
|
|
|
def get_lean_options(args):
|
|
args.lean_options = []
|
|
if args.flycheck:
|
|
args.lean_options.append("--flycheck")
|
|
if args.discard:
|
|
args.lean_options.append("--discard")
|
|
if args.memory:
|
|
args.lean_options += ["-M", args.memory]
|
|
if args.trust:
|
|
args.lean_options += ["-t", args.trust]
|
|
if args.to_axiom:
|
|
args.lean_options.append("--to_axiom")
|
|
if args.cache:
|
|
args.lean_options += ["-c", args.cache]
|
|
if args.lean_config_option:
|
|
for item in args.lean_config_option:
|
|
args.lean_options.append("-D" + item)
|
|
return args
|
|
|
|
def parse_arg(argv):
|
|
parser = argparse.ArgumentParser(description='linja: ninja build wrapper for Lean theorem prover.')
|
|
parser.add_argument('--flycheck', '-F', action='store_true', default=False, help="Use --flycheck option for Lean.")
|
|
parser.add_argument('--flycheck-max-messages', action='store', type=int, default=None, const=999999, nargs='?', help="Number of maximum flycheck messages to display.")
|
|
parser.add_argument('--cache', action='store', help="Use specified cache (clean) file.")
|
|
parser.add_argument('--directory', '-C', action='store', help="change to DIR before doing anything else.")
|
|
parser.add_argument('--lean-config-option', '-D', action='append', help="set a Lean configuration option (name=value)")
|
|
parser.add_argument('--verbose', '-v', action='store_true', help="turn on verbose option")
|
|
parser.add_argument('--memory', '-M', action='store', default=None, const=1, nargs='?', help="maximum amount of memory that can be used by Lean [default=unbounded]")
|
|
parser.add_argument('--keep-going', '-k', action='store', default=None, const=1, nargs='?', help="keep going until N jobs fail [default=1]")
|
|
parser.add_argument('--discard', '-r', action='store_true', default=False, help="discard the proof of imported theorems after checking")
|
|
parser.add_argument('--to_axiom', '-X', action='store_true', default=False, help="discard proofs of all theorems after checking them, i.e., theorems become axioms after checking")
|
|
parser.add_argument('--trust', '-t', action='store_true', default=False, help="trust level [default: max]")
|
|
parser.add_argument('targets', nargs='*')
|
|
args = parser.parse_args(argv)
|
|
check_requirements()
|
|
args = process_args(args)
|
|
args = get_lean_options(args)
|
|
return args
|
|
|
|
def debug_status(args):
|
|
print("Working Directory =", os.getcwd())
|
|
print("")
|
|
for key, val in iteritems(vars(args)):
|
|
print("Option[" + key + "] =", val)
|
|
print("")
|
|
print("linja path =", g_linja_path)
|
|
print("lean/bin dir =", g_lean_bin_dir)
|
|
print("phony targets =", g_phony_targets)
|
|
print("lean path =", g_lean_path)
|
|
print("leantags path =", g_leantags_path)
|
|
print("ninja path =", g_ninja_path)
|
|
|
|
def handle_flycheck_failure(out, err, args):
|
|
if len(args.targets) == 0:
|
|
error("handle_flycheck_failure is called without targets")
|
|
target = args.targets[0]
|
|
failed = set()
|
|
for line in out.splitlines():
|
|
if line.startswith("FAILED:"):
|
|
for lean_file in find_lean_files(args):
|
|
lean = lean_file['lean']
|
|
if lean in line and lean != target:
|
|
failed.add(lean)
|
|
for failed_file in failed:
|
|
print(g_flycheck_header, "ERROR")
|
|
print("%s:1:0: error: failed to compile %s" % (target, failed_file))
|
|
print(g_flycheck_footer)
|
|
if err:
|
|
print(g_flycheck_header, "ERROR")
|
|
print("%s:1:0: error: %s" % (target, err.strip()))
|
|
print(g_flycheck_footer)
|
|
if failed:
|
|
call_lean(target, args)
|
|
|
|
def process_lean_output(target, out, args, using_hlean):
|
|
n = args.flycheck_max_messages
|
|
if target.endswith(".olean"):
|
|
if using_hlean:
|
|
target = target[:-5] + "hlean"
|
|
else:
|
|
target = target[:-5] + "lean"
|
|
if (not target.endswith(".lean") and not using_hlean) or (not target.endswith(".hlean") and using_hlean):
|
|
print(out)
|
|
return
|
|
# Parse, filter, and remove extra items
|
|
flycheckItemList = FlycheckItemList.fromString(target, out)
|
|
flycheckItemList.filter(lambda item: item.filename == target)
|
|
flycheckItemList.removeExtraItemsStartswith("failed to add declaration")
|
|
# Only keep n items in the list.
|
|
# Add tooManyItemsError at the end if we truncated the list
|
|
if n and len(flycheckItemList) > n:
|
|
count = len(flycheckItemList)
|
|
flycheckItemList.truncate(n)
|
|
tooManyItemsError = FlycheckItem(target, 1, 0, "error", "For performance, we only display %d errors/warnings out of %d. (lean-flycheck-max-messages-to-display)" % (n, count))
|
|
flycheckItemList.append(tooManyItemsError)
|
|
print(flycheckItemList)
|
|
|
|
def call_ninja(args):
|
|
targets = []
|
|
for item in args.targets:
|
|
if item.endswith(".lean"):
|
|
targets.append(item[:-4] + "olean")
|
|
elif item.endswith(".hlean"):
|
|
targets.append(item[:-5] + "olean")
|
|
else:
|
|
targets.append(item)
|
|
proc_out = proc_err = None
|
|
if args.flycheck:
|
|
proc_out = subprocess.PIPE
|
|
proc_err = subprocess.PIPE
|
|
ninja_option = []
|
|
if args.keep_going:
|
|
ninja_option += ["-k", args.keep_going]
|
|
proc = subprocess.Popen([g_ninja_path] + ninja_option + targets, stdout=proc_out, stderr=proc_err)
|
|
(out, err) = proc.communicate()
|
|
if args.flycheck:
|
|
if len(args.targets) == 1 and (args.targets[0].endswith(".lean") or args.targets[0].endswith(".hlean")):
|
|
process_lean_output(targets[0], out, args, args.targets[0].endswith(".hlean"))
|
|
handle_flycheck_failure(out, err, args)
|
|
else:
|
|
print(out + err)
|
|
|
|
return proc.returncode
|
|
|
|
def call_lean(filename, args):
|
|
proc = subprocess.Popen([g_lean_path] + args.lean_options + [filename],
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
out = proc.communicate()[0]
|
|
process_lean_output(filename, out, args, filename.endswith(".hlean"))
|
|
return proc.returncode
|
|
|
|
def get_lean_names(lean_file, args, using_hlean):
|
|
lean_file = os.path.abspath(lean_file)
|
|
basename, ext = os.path.splitext(lean_file)
|
|
basename = normalize_drive_name(basename)
|
|
item = {"base" : basename}
|
|
lean_name_exts = ["lean", "olean", "clean", "ilean", "d"]
|
|
for ext in lean_name_exts:
|
|
if ext == "lean" and using_hlean:
|
|
item[ext] = basename + ".hlean"
|
|
else:
|
|
item[ext] = basename + "." + ext
|
|
if args.cache and len(args.targets) == 1 and item['lean'] == args.targets[0]:
|
|
item['clean'] = args.cache
|
|
return item
|
|
|
|
def find_files(directory, pattern):
|
|
if "/" in pattern:
|
|
return glob.glob(os.path.join(directory, pattern))
|
|
matches = []
|
|
for root, dirnames, filenames in os.walk(directory):
|
|
for filename in fnmatch.filter(filenames, pattern):
|
|
matches.append(normalize_drive_name(os.path.join(root, filename)))
|
|
return matches
|
|
|
|
def find_lean_files(args):
|
|
"""Find lean files under project directory. Include and exclude
|
|
files based on patterns in .project file. Use static cache to
|
|
compute only once and reuse it"""
|
|
project_dir = args.project_dir
|
|
if find_lean_files.cached_list != []:
|
|
return find_lean_files.cached_list
|
|
files = set()
|
|
include_patterns, exclude_patterns = [], []
|
|
with open(os.path.join(project_dir, g_project_filename), 'r') as f:
|
|
for line in f:
|
|
c = line[0]
|
|
if c == '+':
|
|
include_patterns.append(line[1:].strip())
|
|
elif c == '-':
|
|
exclude_patterns.append(line[1:].strip())
|
|
elif c == '#':
|
|
pass # Comment
|
|
elif c == 'T':
|
|
pass # TAG
|
|
elif c == 'O':
|
|
pass # Lean Option
|
|
for pattern in include_patterns:
|
|
files |= set(find_files(project_dir, pattern))
|
|
for pattern in exclude_patterns:
|
|
files -= set(find_files(project_dir, pattern))
|
|
has_lean = False
|
|
has_hlean = False
|
|
for file in files:
|
|
if file.endswith(".lean"):
|
|
has_lean = True
|
|
if file.endswith(".hlean"):
|
|
has_hlean = True
|
|
if has_lean and has_hlean:
|
|
error("project cannot mix .lean and .hlean files")
|
|
for file in args.targets:
|
|
if file.endswith(".lean") or file.endswith(".hlean"):
|
|
files.add(file)
|
|
elif file.endswith(".olean"):
|
|
if has_hlean:
|
|
file.add(file[:-5] + "hlean")
|
|
else:
|
|
file.add(file[:-5] + "lean")
|
|
for f in files:
|
|
find_lean_files.cached_list.append(get_lean_names(f, args, has_hlean))
|
|
return find_lean_files.cached_list
|
|
|
|
# Initialize static variable
|
|
find_lean_files.cached_list = []
|
|
|
|
def clear_cache(args):
|
|
files = find_lean_files(args)
|
|
files = find_lean_files(args)
|
|
files = find_lean_files(args)
|
|
files = find_lean_files(args)
|
|
num_of_files = len(files)
|
|
i = 0
|
|
for item in files:
|
|
i += 1
|
|
clean_file = item['clean']
|
|
if os.path.isfile(clean_file):
|
|
sys.stderr.write("[%i/%i] clear cache... % -80s\r" % (i, num_of_files, clean_file))
|
|
sys.stderr.flush()
|
|
os.remove(clean_file)
|
|
if num_of_files > 0:
|
|
sys.stderr.write("\n")
|
|
|
|
def build_olean(lean, olean, clean, dlean, ilean, base):
|
|
(lean, olean, clean, dlean, ilean, base) = list(map(escape_ninja_char, (lean, olean, clean, dlean, ilean, base)))
|
|
if clean.startswith(base):
|
|
str = """build %s %s %s: LEAN %s | %s""" % (olean, ilean, clean, lean, dlean)
|
|
else:
|
|
str = """build %s %s: LEAN %s | %s""" % (olean, ilean, lean, dlean)
|
|
if g_lean_bin_dep_flag:
|
|
str += " %s" % escape_ninja_char(normalize_drive_name(g_lean_path))
|
|
str += "\n"
|
|
str += " DLEAN_FILE=%s\n" % dlean
|
|
str += " OLEAN_FILE=%s\n" % olean
|
|
str += " CLEAN_FILE=%s\n" % clean
|
|
str += " ILEAN_FILE=%s\n" % ilean
|
|
return str
|
|
|
|
def make_build_ninja(args):
|
|
with open(os.path.join(args.project_dir, "build.ninja"), "w") as f:
|
|
lean_files = find_lean_files(args)
|
|
|
|
print("""rule CLEAN""", file=f)
|
|
print(""" command = """, end=' ', file=f)
|
|
print(""""%s" -t clean""" % g_ninja_path, file=f)
|
|
print(""" description = Cleaning all built files...""", file=f)
|
|
|
|
print("""rule LEAN""", file=f)
|
|
print(""" depfile = ${DLEAN_FILE}""", file=f)
|
|
print(""" command = "%s" %s $in -o "${OLEAN_FILE}" -c "${CLEAN_FILE}" -i "${ILEAN_FILE}" """ \
|
|
% (g_lean_path, " ".join(args.lean_options)), file=f)
|
|
|
|
print("""rule LEANTAGS""", file=f)
|
|
print(""" command = """, end=' ', file=f)
|
|
if platform.system() == "Windows":
|
|
print("python ", end=' ', file=f)
|
|
print(""""%s" --relative -- $in """ % (g_leantags_path), file=f)
|
|
|
|
print("build all: phony", end=' ', file=f)
|
|
for item in lean_files:
|
|
print(" " + escape_ninja_char(item['olean']), end=' ', file=f)
|
|
print("", file=f)
|
|
|
|
tags_file = "TAGS"
|
|
print("build tags: phony " + tags_file, file=f)
|
|
print("build " + tags_file + ": LEANTAGS", end=' ', file=f)
|
|
for item in lean_files:
|
|
print(" " + escape_ninja_char(item['ilean']), end=' ', file=f)
|
|
print("", file=f)
|
|
|
|
print("""build clean: CLEAN""", file=f)
|
|
|
|
for item in lean_files:
|
|
print(build_olean(item['lean'], item['olean'], item['clean'], item['d'], item['ilean'], item['base']), file=f)
|
|
|
|
print("""default all""", file=f)
|
|
|
|
def escape_dep(s):
|
|
return s.replace(" ", "\\ ")
|
|
|
|
def make_deps(lean_file, dlean_file, olean_file):
|
|
with open(dlean_file, "w") as f:
|
|
deps = []
|
|
proc = subprocess.Popen([g_lean_path, "--deps", lean_file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
output = proc.communicate()[0]
|
|
print(escape_dep(olean_file) + ": \\", file=f)
|
|
for olean_file in output.strip().splitlines():
|
|
if olean_file:
|
|
deps.append(normalize_drive_name(os.path.abspath(olean_file)))
|
|
deps = list(map(escape_dep, deps))
|
|
deps_str = b" " + (b" \\\n ".join(deps))
|
|
print(deps_str, file=f)
|
|
|
|
def make_deps_all_files(args):
|
|
lean_files = find_lean_files(args)
|
|
threads = []
|
|
i, num_of_files = 1, len(lean_files)
|
|
for item in lean_files:
|
|
lean_file = item['lean']
|
|
dlean_file = item['d']
|
|
olean_file = item['olean']
|
|
if not os.path.isfile(dlean_file) or os.path.getmtime(dlean_file) < os.path.getmtime(lean_file):
|
|
thread = threading.Thread(target=make_deps, args = [lean_file, dlean_file, olean_file])
|
|
sys.stderr.write("[%i/%i] generating dep... % -80s" % (i, num_of_files, dlean_file))
|
|
if args.flycheck == True:
|
|
sys.stderr.write("\n")
|
|
else:
|
|
sys.stderr.write("\r")
|
|
sys.stderr.flush()
|
|
thread.start()
|
|
threads.append(thread)
|
|
i += 1
|
|
for thread in threads:
|
|
thread.join()
|
|
if threads != []:
|
|
sys.stderr.write("\n")
|
|
|
|
def main(argv=None):
|
|
init_logger()
|
|
if argv is None:
|
|
argv = sys.argv[1:]
|
|
args = parse_arg(argv)
|
|
if args.project_dir:
|
|
if args.targets == ["clear-cache"]:
|
|
args.targets = []
|
|
clear_cache(args)
|
|
return 0
|
|
if not "clean" in args.targets:
|
|
make_deps_all_files(args)
|
|
make_build_ninja(args)
|
|
return call_ninja(args)
|
|
|
|
else: # NO Project Directory Found
|
|
if args.phony_targets:
|
|
error("cannot find project directory. Make sure that you have " \
|
|
+ g_project_filename + " file at the project root.")
|
|
returncode = 0
|
|
for filename in args.targets:
|
|
if os.path.isfile(filename) and (filename.endswith(".lean") or filename.endswith(".hlean")):
|
|
returncode |= call_lean(filename, args)
|
|
return returncode
|
|
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|