feat(frontends/lean): add 'wait' command

This commit also fixes several problems with -j option (parallel
compilation). The .olean files were not missing data when -j was used

see issue #576

src/emacs/lean-syntax.el

@@ -16,7 +16,7 @@
     "alias" "help" "environment" "options" "precedence" "reserve"
     "match" "infix" "infixl" "infixr" "notation" "postfix" "prefix"
     "tactic_infix" "tactic_infixl" "tactic_infixr" "tactic_notation" "tactic_postfix" "tactic_prefix"
-    "eval" "check" "coercion" "end"
+    "eval" "check" "coercion" "end" "wait"
     "using" "namespace" "section" "fields" "find_decl"
     "attribute" "local" "set_option" "add_rewrite" "extends" "include" "omit" "classes"
     "instances" "coercions" "metaclasses" "raw" "migrate" "replacing")

src/frontends/lean/builtin_cmds.cpp

@@ -173,6 +173,9 @@ static void print_definition(parser const & p, name const & n, pos_info const &
     options opts        = out.get_options();
     opts                = opts.update_if_undef(get_pp_beta_name(), false);
     io_state_stream new_out = out.update_options(opts);
+    if (d.is_axiom())
+        throw parser_error(sstream() << "invalid 'print definition', theorem '" << n
+                           << "' is not available (suggestion: use command 'wait " << n << "')", pos);
     if (!d.is_definition())
         throw parser_error(sstream() << "invalid 'print definition', '" << n << "' is not a definition", pos);
     new_out << d.get_value() << endl;

src/frontends/lean/decl_cmds.cpp

@@ -1058,16 +1058,19 @@ class definition_cmd_fn {
                     c_value = expand_abbreviations(m_env, unfold_untrusted_macros(m_env, c_value));
                     if (m_kind == Theorem) {
                         cd = check(mk_theorem(m_real_name, c_ls, c_type, c_value));
-                        if (!m_p.keep_new_thms()) {
+                        if (m_p.keep_new_thms()) {
-                            // discard theorem
+                            if (!m_is_private)
-                            cd = check(mk_axiom(m_real_name, c_ls, c_type));
+                                m_p.add_decl_index(m_real_name, m_pos, m_p.get_cmd_token(), c_type);
+                            m_p.add_delayed_theorem(*cd);
                         }
+                        cd = check(mk_axiom(m_real_name, c_ls, c_type));
+                        m_env = module::add(m_env, *cd);
                     } else {
                         cd = check(mk_definition(m_env, m_real_name, c_ls, c_type, c_value));
+                        if (!m_is_private)
+                            m_p.add_decl_index(m_real_name, m_pos, m_p.get_cmd_token(), c_type);
+                        m_env = module::add(m_env, *cd);
                     }
-                    if (!m_is_private)
-                        m_p.add_decl_index(m_real_name, m_pos, m_p.get_cmd_token(), c_type);
-                    m_env = module::add(m_env, *cd);
                     return true;
                 } catch (exception&) {}
             }
@@ -1217,10 +1220,10 @@ class definition_cmd_fn {
                     auto cd = check(mk_theorem(m_real_name, new_ls, m_type, m_value));
                     if (m_kind == Theorem) {
                         // Remark: we don't keep examples
-                        if (!m_p.keep_new_thms()) {
+                        if (m_p.keep_new_thms()) {
-                            // discard theorem
+                            m_p.add_delayed_theorem(cd);
-                            cd = check(mk_axiom(m_real_name, new_ls, m_type));
                         }
+                        cd = check(mk_axiom(m_real_name, new_ls, m_type));
                         m_env = module::add(m_env, cd);
                         m_p.cache_definition(m_real_name, pre_type, pre_value, new_ls, m_type, m_value);
                     }
@@ -1407,6 +1410,16 @@ environment local_attribute_cmd(parser & p) {
     return attribute_cmd_core(p, false);
 }
 
+static environment wait_cmd(parser & p) {
+    buffer<name> ds;
+    name d          = p.check_constant_next("invalid 'wait' command, constant expected");
+    ds.push_back(d);
+    while (p.curr_is_identifier()) {
+        ds.push_back(p.check_constant_next("invalid 'wait' command, constant expected"));
+    }
+    return p.wait_theorems(ds);
+}
+
 void register_decl_cmds(cmd_table & r) {
     add_cmd(r, cmd_info("universe",     "declare a universe level", universe_cmd));
     add_cmd(r, cmd_info("universes",    "declare universe levels", universes_cmd));
@@ -1423,6 +1436,7 @@ void register_decl_cmds(cmd_table & r) {
     add_cmd(r, cmd_info("private",      "add new private definition/theorem", private_definition_cmd));
     add_cmd(r, cmd_info("protected",    "add new protected definition/theorem", protected_definition_cmd));
     add_cmd(r, cmd_info("theorem",      "add new theorem", theorem_cmd));
+    add_cmd(r, cmd_info("wait",         "wait for theorems to be processed", wait_cmd));
     add_cmd(r, cmd_info("include",      "force section parameter/variable to be included", include_cmd));
     add_cmd(r, cmd_info("attribute",    "set declaration attributes", attribute_cmd));
     add_cmd(r, cmd_info("abbreviation", "declare a new abbreviation", abbreviation_cmd));

src/frontends/lean/parser.cpp

@@ -1709,7 +1709,7 @@ void parser::parse_imports() {
     for (auto const & f : lua_files) {
         std::string rname = find_file(f, {".lua"});
         system_import(rname.c_str());
-        m_env = module::add(m_env, *g_lua_module_key, [=](serializer & s) {
+        m_env = module::add(m_env, *g_lua_module_key, [=](environment const &, serializer & s) {
                 s << f;
             });
     }
@@ -1786,10 +1786,14 @@ bool parser::parse_commands() {
             m_env = pop_scope_core(m_env, m_ios);
     }
     commit_info(m_scanner.get_line()+1, 0);
-    for (certified_declaration const & thm : m_theorem_queue.join()) {
+
-        if (keep_new_thms())
+    m_theorem_queue.for_each([&](certified_declaration const & thm) {
-            m_env.replace(thm);
+            if (keep_new_thms()) {
-    }
+                name const & thm_name = thm.get_declaration().get_name();
+                if (m_env.get(thm_name).is_axiom())
+                    m_env = m_env.replace(thm);
+            }
+        });
     return !m_found_errors;
 }
 
@@ -1813,6 +1817,23 @@ void parser::add_delayed_theorem(environment const & env, name const & n, level_
     m_theorem_queue.add(env, n, ls, get_local_level_decls(), t, v);
 }
 
+void parser::add_delayed_theorem(certified_declaration const & cd) {
+    m_theorem_queue.add(cd);
+}
+
+environment parser::wait_theorems(buffer<name> const & ds) {
+    m_theorem_queue.for_each([&](certified_declaration const & thm) {
+            if (keep_new_thms()) {
+                name const & thm_name = thm.get_declaration().get_name();
+                if (m_env.get(thm_name).is_axiom() &&
+                    std::any_of(ds.begin(), ds.end(), [&](name const & n) { return n == thm_name; })) {
+                    m_env = m_env.replace(thm);
+                }
+            }
+        });
+    return m_env;
+}
+
 void parser::save_snapshot() {
     m_pre_info_manager.clear();
     if (!m_snapshot_vector)

src/frontends/lean/parser.h

@@ -300,6 +300,8 @@ public:
 
     unsigned num_threads() const { return m_num_threads; }
     void add_delayed_theorem(environment const & env, name const & n, level_param_names const & ls, expr const & t, expr const & v);
+    void add_delayed_theorem(certified_declaration const & cd);
+    environment wait_theorems(buffer<name> const & ds);
 
     /** \brief Read the next token. */
     void scan() { m_curr = m_scanner.scan(m_env); }

src/frontends/lean/theorem_queue.cpp

@@ -12,10 +12,20 @@ Author: Leonardo de Moura
 #include "frontends/lean/parser.h"
 
 namespace lean {
-theorem_queue::theorem_queue(parser & p, unsigned num_threads):m_parser(p), m_queue(num_threads, []() { enable_expr_caching(false); }) {}
+void theorem_queue::init_queue() {
+    m_queue.reset(new worker_queue<certified_declaration>(m_num_threads, []() { enable_expr_caching(false); }));
+}
+void theorem_queue::consume() {
+    for (auto const & c : m_queue->join())
+        m_ready.push_back(c);
+    init_queue();
+}
+theorem_queue::theorem_queue(parser & p, unsigned num_threads):m_parser(p), m_num_threads(num_threads) {
+    init_queue();
+}
 void theorem_queue::add(environment const & env, name const & n, level_param_names const & ls, local_level_decls const & lls,
                         expr const & t, expr const & v) {
-    m_queue.add([=]() {
+    m_queue->add([=]() {
             level_param_names new_ls;
             expr type, value;
             std::tie(type, value, new_ls) = m_parser.elaborate_definition_at(env, lls, n, t, v);
@@ -26,7 +36,15 @@ void theorem_queue::add(environment const & env, name const & n, level_param_nam
             return r;
         });
 }
-std::vector<certified_declaration> const & theorem_queue::join() { return m_queue.join(); }
+void theorem_queue::add(certified_declaration const & c) {
-void theorem_queue::interrupt() { m_queue.interrupt(); }
+    m_ready.push_back(c);
-bool theorem_queue::done() const { return m_queue.done(); }
+}
+void theorem_queue::for_each(std::function<void(certified_declaration const & c)> const & fn) {
+    consume();
+    for (auto const & c : m_ready)
+        fn(c);
+}
+void theorem_queue::join() { m_queue->join(); }
+void theorem_queue::interrupt() { m_queue->interrupt(); }
+bool theorem_queue::done() const { return m_queue->done(); }
 }

src/frontends/lean/theorem_queue.h

@@ -15,12 +15,18 @@ class parser;
 typedef local_decls<level>  local_level_decls;
 class theorem_queue {
     parser & m_parser;
-    worker_queue<certified_declaration> m_queue;
+    unsigned m_num_threads;
+    std::unique_ptr<worker_queue<certified_declaration>> m_queue;
+    std::vector<certified_declaration>  m_ready;
+    void init_queue();
+    void consume();
 public:
     theorem_queue(parser & p, unsigned num_threads);
     void add(environment const & env, name const & n, level_param_names const & ls, local_level_decls const & lls,
              expr const & t, expr const & v);
-    std::vector<certified_declaration> const & join();
+    void add(certified_declaration const & c);
+    void for_each(std::function<void(certified_declaration const & c)> const & fn);
+    void join();
     void interrupt();
     bool done() const;
 };

src/frontends/lean/token_table.cpp

@@ -103,7 +103,7 @@ void init_token_table(token_table & t) {
          {"<d", g_decreasing_prec}, {"renaming", 0}, {"extends", 0}, {nullptr, 0}};
 
     char const * commands[] =
-        {"theorem", "axiom", "axioms", "variable", "protected", "private",
+        {"theorem", "axiom", "axioms", "variable", "protected", "private", "wait",
          "definition", "example", "coercion", "abbreviation",
          "variables", "parameter", "parameters", "constant", "constants", "[persistent]", "[visible]", "[instance]",
          "[none]", "[class]", "[coercion]", "[reducible]", "[irreducible]", "[semireducible]", "[quasireducible]",

src/library/module.cpp

@@ -37,7 +37,7 @@ corrupted_file_exception::corrupted_file_exception(std::string const & fname):
     exception(sstream() << "failed to import '" << fname << "', file is corrupted, please regenerate the file from sources") {
 }
 
-typedef pair<std::string, std::function<void(serializer &)>> writer;
+typedef pair<std::string, std::function<void(environment const &, serializer &)>> writer;
 
 struct module_ext : public environment_extension {
     list<module_name> m_direct_imports;
@@ -146,7 +146,7 @@ void export_module(std::ostream & out, environment const & env) {
     // store objects
     for (auto p : writers) {
         s1 << p->first;
-        p->second(s1);
+        p->second(env, s1);
     }
     s1 << g_olean_end_file;
 
@@ -182,7 +182,7 @@ static std::string * g_quotient  = nullptr;
 static std::string * g_hits      = nullptr;
 
 namespace module {
-environment add(environment const & env, std::string const & k, std::function<void(serializer &)> const & wr) {
+environment add(environment const & env, std::string const & k, std::function<void(environment const &, serializer &)> const & wr) {
     module_ext ext = get_extension(env);
     ext.m_writers  = cons(writer(k, wr), ext.m_writers);
     return update(env, ext);
@@ -193,7 +193,7 @@ environment add_universe(environment const & env, name const & l) {
     module_ext ext = get_extension(env);
     ext.m_module_univs = cons(l, ext.m_module_univs);
     new_env = update(new_env, ext);
-    return add(new_env, *g_glvl_key, [=](serializer & s) { s << l; });
+    return add(new_env, *g_glvl_key, [=](environment const &, serializer & s) { s << l; });
 }
 
 environment update_module_defs(environment const & env, declaration const & d) {
@@ -209,17 +209,22 @@ environment update_module_defs(environment const & env, declaration const & d) {
     }
 }
 
+static environment export_decl(environment const & env, declaration const & d) {
+    name n = d.get_name();
+    return add(env, *g_decl_key, [=](environment const & env, serializer & s) {
+            s << env.get(n);
+        });
+}
+
 environment add(environment const & env, certified_declaration const & d) {
     environment new_env = env.add(d);
     declaration _d = d.get_declaration();
-    new_env = update_module_defs(new_env, _d);
+    return export_decl(update_module_defs(new_env, _d), _d);
-    return add(new_env, *g_decl_key, [=](serializer & s) { s << _d; });
 }
 
 environment add(environment const & env, declaration const & d) {
     environment new_env = env.add(d);
-    new_env = update_module_defs(new_env, d);
+    return export_decl(update_module_defs(new_env, d), d);
-    return add(new_env, *g_decl_key, [=](serializer & s) { s << d; });
 }
 
 bool is_definition(environment const & env, name const & n) {
@@ -229,7 +234,7 @@ bool is_definition(environment const & env, name const & n) {
 
 environment declare_quotient(environment const & env) {
     environment new_env = ::lean::declare_quotient(env);
-    return add(new_env, *g_quotient, [=](serializer &) {});
+    return add(new_env, *g_quotient, [=](environment const &, serializer &) {});
 }
 
 static void quotient_reader(deserializer &, shared_environment & senv,
@@ -242,7 +247,7 @@ static void quotient_reader(deserializer &, shared_environment & senv,
 
 environment declare_hits(environment const & env) {
     environment new_env = ::lean::declare_hits(env);
-    return add(new_env, *g_hits, [=](serializer &) {});
+    return add(new_env, *g_hits, [=](environment const &, serializer &) {});
 }
 
 static void hits_reader(deserializer &, shared_environment & senv,
@@ -261,7 +266,7 @@ environment add_inductive(environment                  env,
     module_ext ext = get_extension(env);
     ext.m_module_decls = cons(inductive::inductive_decl_name(head(decls)), ext.m_module_decls);
     new_env = update(new_env, ext);
-    return add(new_env, *g_inductive, [=](serializer & s) {
+    return add(new_env, *g_inductive, [=](environment const &, serializer & s) {
             s << inductive_decls(level_params, num_params, decls);
         });
 }

src/library/module.h

@@ -92,7 +92,7 @@ namespace module {
 
     \see module_object_reader
 */
-environment add(environment const & env, std::string const & k, std::function<void(serializer &)> const & writer);
+environment add(environment const & env, std::string const & k, std::function<void(environment const &, serializer &)> const & writer);
 
 /** \brief Add the global universe declaration to the environment, and mark it to be exported. */
 environment add_universe(environment const & env, name const & l);

src/library/private.cpp

@@ -37,7 +37,7 @@ static std::string * g_prv_key = nullptr;
 // Make sure the mapping "hidden-name r ==> user-name n" is preserved when we close sections and
 // export .olean files.
 static environment preserve_private_data(environment const & env, name const & r, name const & n) {
-    return module::add(env, *g_prv_key, [=](serializer & s) { s << n << r; });
+    return module::add(env, *g_prv_key, [=](environment const &, serializer & s) { s << n << r; });
 }
 
 pair<environment, name> add_private_name(environment const & env, name const & n, optional<unsigned> const & extra_hash) {

src/library/projection.cpp

@@ -48,7 +48,9 @@ static environment save_projection_info_core(environment const & env, name const
 
 environment save_projection_info(environment const & env, name const & p, name const & mk, unsigned nparams, unsigned i, bool inst_implicit) {
     environment new_env = save_projection_info_core(env, p, mk, nparams, i, inst_implicit);
-    return module::add(new_env, *g_proj_key, [=](serializer & s) { s << p << mk << nparams << i << inst_implicit; });
+    return module::add(new_env, *g_proj_key, [=](environment const &, serializer & s) {
+            s << p << mk << nparams << i << inst_implicit;
+        });
 }
 
 projection_info const * get_projection_info(environment const & env, name const & p) {

src/library/protected.cpp

@@ -34,7 +34,7 @@ environment add_protected(environment const & env, name const & n) {
     protected_ext ext = get_extension(env);
     ext.m_protected.insert(n);
     environment new_env = update(env, ext);
-    return module::add(new_env, *g_prt_key, [=](serializer & s) { s << n; });
+    return module::add(new_env, *g_prt_key, [=](environment const &, serializer & s) { s << n; });
 }
 
 static void protected_reader(deserializer & d, shared_environment & senv,

src/library/scoped_ext.cpp

@@ -127,7 +127,7 @@ environment add_namespace(environment const & env, name const & ns) {
     if (!ext.m_namespace_set.contains(ns)) {
         ext.m_namespace_set.insert(ns);
         environment r = update(env, ext);
-        return module::add(r, *g_new_namespace_key, [=](serializer & s) { s << ns; });
+        return module::add(r, *g_new_namespace_key, [=](environment const &, serializer & s) { s << ns; });
     } else {
         return env;
     }
@@ -155,7 +155,7 @@ environment push_scope(environment const & env, io_state const & ios, scope_kind
     if (k == scope_kind::Namespace)
         r = using_namespace(r, ios, new_n);
     if (save_ns)
-        r = module::add(r, *g_new_namespace_key, [=](serializer & s) { s << new_n; });
+        r = module::add(r, *g_new_namespace_key, [=](environment const &, serializer & s) { s << new_n; });
     return r;
 }
 

src/library/scoped_ext.h

@@ -237,7 +237,7 @@ public:
             return update(env, get(env)._add_tmp_entry(env, ios, e));
         } else {
             name n = get_namespace(env);
-            env = module::add(env, get_serialization_key(), [=](serializer & s) {
+            env = module::add(env, get_serialization_key(), [=](environment const &, serializer & s) {
                     s << n;
                     write_entry(s, e);
                 });

tests/lean/hott/tele_eq.hlean

@@ -14,5 +14,5 @@ begin
   cases H₃,
   apply rfl
 end
-
+wait foo.eq
 print definition foo.eq

tests/lean/run/calc_heq_symm.lean

@@ -7,4 +7,5 @@ calc d  == c  : H₃
     ... == a₂ : H₁
     ... =  a₁ : H₀
 
+wait tst
 print definition tst

tests/lean/run/class2.lean

@@ -3,7 +3,7 @@ open prod nonempty inhabited
 
 theorem H {A B : Type} (H1 : inhabited A) : inhabited (Prop × A × (B → num))
 := _
-
+wait H
 (*
 print(get_env():find("H"):value())
 *)

tests/lean/run/class3.lean

@@ -11,6 +11,7 @@ section
 
 end
 
+wait tst
 (*
 print(get_env():find("tst"):value())
 *)

tests/lean/run/class8.lean

@@ -32,6 +32,7 @@ have h1 [visible] : inh A, from inh.intro a,
 have h2 [visible] : inh C, from inh_exists H2,
 _
 
+wait T1
 (*
 print(get_env():find("T1"):value())
 *)

tests/lean/run/dfun_tst.lean

@@ -5,6 +5,7 @@ theorem tst1 : inhabited (vector nat 2)
 
 theorem tst2 : inhabited (Prop × (Π n : nat, vector nat n))
 
+wait tst2
 (*
 print(get_env():find("tst2"):value())
 *)

tests/lean/run/local_eqns.lean

@@ -64,4 +64,5 @@ theorem add_eq_addl : ∀ x y, x + y = x ⊕ y
     rewrite [s_add, add_eq_addl]
   end
 
+wait add_eq_addl
 print definition add_eq_addl

tests/lean/run/match_tac3.lean

@@ -10,4 +10,5 @@ begin
   end
 end
 
+wait tst
 print definition tst

tests/lean/run/match_tac4.lean

@@ -22,6 +22,7 @@ begin
   end
 end
 
+wait tst
 print definition tst
 
 theorem tst2 (a b c d : Prop) : a ∧ b ∧ c ∧ d ↔ d ∧ c ∧ b ∧ a :=
@@ -30,4 +31,5 @@ begin
   repeat (intro H;  repeat (cases H with [H', H] | apply and.intro | assumption))
 end
 
+wait tst2
 print definition tst2

tests/lean/run/revert_tac.lean

@@ -23,5 +23,7 @@ begin
   apply (eq.rec_on Heq Hp)
 end
 
+wait foo1 foo2
+
 print definition foo1
 print definition foo2

tests/lean/run/reverts_tac.lean

@@ -24,5 +24,6 @@ begin
   apply (eq.rec_on Heq Hp)
 end
 
+wait foo1 foo2
 print definition foo1
 print definition foo2

tests/lean/run/rewriter1.lean

@@ -6,4 +6,5 @@ begin
   rewrite [add.assoc, {b + _}add.comm, -add.assoc]
 end
 
+wait test
 print definition test

tests/lean/run/rewriter2.lean

@@ -25,6 +25,7 @@ begin
   rewrite [+mul_zero, +zero_mul, +add_zero] -- in rewrite rules, + is notation for one or more
 end
 
+wait test3
 print definition test3
 
 theorem test4 {A : Type} [s : comm_ring A] (a b c : A) : a * 0 + 0 * b + c * 0 + 0 * a = 0 :=

tests/lean/run/tactic21.lean

@@ -7,6 +7,7 @@ theorem tst1 {A : Type} {a b c : A} {p : A → A → Prop} (H1 : p a b) (H2 : p
 theorem tst2 {A : Type} {a b c d : A} {p : A → A → Prop} (Ha : p a c) (H1 : p a b) (Hb : p b d) (H2 : p b c) : ∃ x, p a x ∧ p x c
 := by apply exists.intro; apply and.intro; eassumption; eassumption
 
+wait tst2
 (*
 print(get_env():find("tst2"):value())
 *)

tests/lean/run/tele_eq.lean

@@ -15,4 +15,5 @@ begin
   apply rfl
 end
 
+wait foo.eq
 print definition foo.eq