diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index c2f52d0fa9faf..1952da6fea3da 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -14564,6 +14564,21 @@ Evaluate the contents of a source file in the current context. During including, """ include +doc""" +```rst +:: + include_dependency(path::AbstractString) + +In a module, declare that the file specified by `path` (relative or +absolute) is a dependency for precompilation; that is, the +module will need to be recompiled if this file changes. + +This is only needed if your module depends on a file that is not +used via `include`. It has no effect outside of compilation. +``` +""" +include_dependency + doc""" ```rst :: diff --git a/base/exports.jl b/base/exports.jl index 81815305b651e..55d20fd323891 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1104,6 +1104,7 @@ export evalfile, include, include_string, + include_dependency, # RTS internals finalizer, diff --git a/base/loading.jl b/base/loading.jl index b8005e7bcc252..f46a4cd68999a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -104,9 +104,30 @@ end const package_locks = Dict{Symbol,Condition}() const package_loaded = Set{Symbol}() +# used to optionally track dependencies when requiring a module: +const _require_dependencies = ByteString[] +const _track_dependencies = [false] +function _include_dependency(_path::AbstractString) + prev = source_path(nothing) + path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path) + if _track_dependencies[1] + push!(_require_dependencies, abspath(path)) + end + return path, prev +end +function include_dependency(path::AbstractString) + _include_dependency(path) + return nothing +end + # require always works in Main scope and loads files from node 1 toplevel_load = true function require(mod::Symbol) + # dependency-tracking is only used for one top-level include(path), + # and is not applied recursively to imported modules: + old_track_dependencies = _track_dependencies[1] + _track_dependencies[1] = false + global toplevel_load loading = get(package_locks, mod, false) if loading !== false @@ -152,6 +173,7 @@ function require(mod::Symbol) toplevel_load = last loading = pop!(package_locks, mod) notify(loading, all=true) + _track_dependencies[1] = old_track_dependencies end nothing end @@ -187,9 +209,8 @@ end macro __FILE__() source_path() end -function include_from_node1(path::AbstractString) - prev = source_path(nothing) - path = (prev === nothing) ? abspath(path) : joinpath(dirname(prev),path) +function include_from_node1(_path::AbstractString) + path, prev = _include_dependency(_path) tls = task_local_storage() tls[:SOURCE_PATH] = path local result @@ -246,6 +267,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString) task_local_storage()[:SOURCE_PATH] = $(source) end) end + serialize(io, :(Base._track_dependencies[1] = true)) serialize(io, :(Base.include($(abspath(input))))) if source !== nothing serialize(io, quote @@ -270,3 +292,26 @@ function compile(name::ByteString) create_expr_cache(path, cachefile) return cachefile end + +module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m) + +function cache_dependencies(cachefile::AbstractString) + modules = Tuple{ByteString,UInt64}[] + files = ByteString[] + open(cachefile, "r") do f + while true + n = ntoh(read(f, Int32)) + n == 0 && break + push!(modules, + (bytestring(readbytes(f, n)), # module name + ntoh(read(f, UInt64)))) # module UUID (timestamp) + end + read(f, Int64) # total bytes for file dependencies + while true + n = ntoh(read(f, Int32)) + n == 0 && break + push!(files, bytestring(readbytes(f, n))) + end + end + return modules, files +end diff --git a/src/dump.c b/src/dump.c index 4adcce633a2a2..bf40bc2c989a0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -132,46 +132,48 @@ static jl_array_t *datatype_list=NULL; // (only used in MODE_SYSTEM_IMAGE) #define write_int8(s, n) write_uint8(s, n) #define read_int8(s) read_uint8(s) +/* read and write in network (bigendian) order: */ + static void write_int32(ios_t *s, int32_t i) { - write_uint8(s, i & 0xff); - write_uint8(s, (i>> 8) & 0xff); - write_uint8(s, (i>>16) & 0xff); write_uint8(s, (i>>24) & 0xff); + write_uint8(s, (i>>16) & 0xff); + write_uint8(s, (i>> 8) & 0xff); + write_uint8(s, i & 0xff); } static int32_t read_int32(ios_t *s) { - int b0 = read_uint8(s); - int b1 = read_uint8(s); - int b2 = read_uint8(s); int b3 = read_uint8(s); + int b2 = read_uint8(s); + int b1 = read_uint8(s); + int b0 = read_uint8(s); return b0 | (b1<<8) | (b2<<16) | (b3<<24); } static void write_uint64(ios_t *s, uint64_t i) { - write_int32(s, i & 0xffffffff); write_int32(s, (i>>32) & 0xffffffff); + write_int32(s, i & 0xffffffff); } static uint64_t read_uint64(ios_t *s) { - uint64_t b0 = (uint32_t)read_int32(s); uint64_t b1 = (uint32_t)read_int32(s); + uint64_t b0 = (uint32_t)read_int32(s); return b0 | (b1<<32); } static void write_uint16(ios_t *s, uint16_t i) { - write_uint8(s, i & 0xff); write_uint8(s, (i>> 8) & 0xff); + write_uint8(s, i & 0xff); } static uint16_t read_uint16(ios_t *s) { - int b0 = read_uint8(s); int b1 = read_uint8(s); + int b0 = read_uint8(s); return b0 | (b1<<8); } @@ -949,6 +951,54 @@ void jl_serialize_mod_list(ios_t *s) write_int32(s, 0); } +// serialize the global _require_dependencies array of pathnames that +// are include depenencies +void jl_serialize_dependency_list(ios_t *s) +{ + size_t total_size = 0; + static jl_array_t *deps = NULL; + if (!deps) + deps = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("_require_dependencies")); + if (deps) { + // sort!(deps) so that we can easily eliminate duplicates + static jl_value_t *sort_func = NULL; + if (!sort_func) + sort_func = jl_get_global(jl_base_module, jl_symbol("sort!")); + jl_apply((jl_function_t*)sort_func, (jl_value_t**)&deps, 1); + + size_t l = jl_array_len(deps); + jl_value_t *prev = NULL; + for (size_t i=0; i < l; i++) { + jl_value_t *dep = jl_cellref(deps, i); + size_t slen = jl_string_len(dep); + if (!prev || + memcmp(jl_string_data(dep), jl_string_data(prev), slen)) { + total_size += 4 + slen; + } + prev = dep; + } + total_size += 4; + } + // write the total size so that we can quickly seek past all of the + // dependencies if we don't need them + write_uint64(s, total_size); + if (deps) { + size_t l = jl_array_len(deps); + jl_value_t *prev = NULL; + for (size_t i=0; i < l; i++) { + jl_value_t *dep = jl_cellref(deps, i); + size_t slen = jl_string_len(dep); + if (!prev || + memcmp(jl_string_data(dep), jl_string_data(prev), slen)) { + write_int32(s, slen); + ios_write(s, jl_string_data(dep), slen); + } + prev = dep; + } + write_int32(s, 0); // terminator, for ease of reading + } +} + // --- deserialize --- static jl_fptr_t jl_deserialize_fptr(ios_t *s) @@ -1893,6 +1943,7 @@ DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) } serializer_worklist = worklist; jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization) + jl_serialize_dependency_list(&f); JL_SIGATOMIC_BEGIN(); arraylist_new(&reinit_list, 0); @@ -1937,6 +1988,8 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) ios_close(f); return NULL; } + size_t deplen = read_uint64(f); + ios_skip(f, deplen); // skip past the dependency list JL_SIGATOMIC_BEGIN(); arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); diff --git a/test/compile.jl b/test/compile.jl index 8fda7e0cae8c9..fdc88860b4a1d 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -13,26 +13,34 @@ try print(f, """ module $Foo_module @doc "foo function" foo(x) = x + 1 + include_dependency("foo.jl") module Bar @doc "bar function" bar(x) = x + 2 + include_dependency("bar.jl") end end """) end - Base.compile(Foo_module) + cachefile = Base.compile(Foo_module) eval(Main, :(import $Foo_module)) + + let Foo = eval(Main, Foo_module) + @test Foo.foo(17) == 18 + @test Foo.Bar.bar(17) == 19 + + # issue #12284: + @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n" + @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" + + deps = Base.cache_dependencies(cachefile) + @test sort(deps[1]) == map(n -> (n, Base.module_uuid(eval(symbol(n)))), + ["Base","Core","Main"]) + @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] + end + finally splice!(Base.LOAD_CACHE_PATH, 1) splice!(LOAD_PATH, 1) rm(dir, recursive=true) end - -let Foo = eval(Main, Foo_module) - @test Foo.foo(17) == 18 - @test Foo.Bar.bar(17) == 19 - - # issue #12284: - @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n" - @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" -end