diff --git a/NEWS.md b/NEWS.md index 3bd56ed843d6b..af23f70a0d3e2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -31,6 +31,12 @@ New language features * The syntax `function foo end` can be used to introduce a generic function without yet adding any methods ([#8283]). + * Incremental compilation of modules: ``Base.compile(module::Symbol)`` (stored in `~/.julia/lib/v0.4`) + + * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. + + * New option `--compile-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line. + Language changes ---------------- @@ -450,6 +456,8 @@ Deprecated or removed * `sync_gc_total_bytes` -> `jl_gc_sync_total_bytes` + * `require(::AbstractString)` and `reload` (see news about addition of `compile`) + Julia v0.3.0 Release Notes ========================== diff --git a/base/boot.jl b/base/boot.jl index b95d541127e42..f96475c39810d 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -71,8 +71,8 @@ # module::Module #end -#type Box{T} -# contents::T +#type Box +# contents::Any #end #abstract Ref{T} diff --git a/base/client.jl b/base/client.jl index 33701242590b4..77005e5f352fe 100644 --- a/base/client.jl +++ b/base/client.jl @@ -274,7 +274,9 @@ let reqarg = Set(UTF8String["--home", "-H", end # load file immediately on all processors if opts.load != C_NULL - require(bytestring(opts.load)) + @sync for p in procs() + @async remotecall_fetch(p, include, bytestring(opts.load)) + end end # eval expression if opts.eval != C_NULL @@ -322,6 +324,7 @@ is_interactive = false isinteractive() = (is_interactive::Bool) const LOAD_PATH = ByteString[] +const LOAD_CACHE_PATH = ByteString[] function init_load_path() vers = "v$(VERSION.major).$(VERSION.minor)" if haskey(ENV,"JULIA_LOAD_PATH") @@ -329,6 +332,8 @@ function init_load_path() end push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers)) push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers)) + push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","lib",vers)) + push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme end function load_juliarc() diff --git a/base/deprecated.jl b/base/deprecated.jl index 0c97fce868e66..90f4f92f0edf6 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -635,7 +635,6 @@ end @deprecate mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Integer}, s::IOStream, offset::FileOffset=position(s)) mmap(s, BitArray, dims, offset) @deprecate mmap_bitarray{N}(dims::NTuple{N,Integer}, s::IOStream, offset=position(s)) mmap(s, BitArray, dims, offset) - # T[a:b] and T[a:s:b] @noinline function getindex{T<:Union{Char,Number}}(::Type{T}, r::Range) depwarn("T[a:b] concatenation is deprecated; use T[a:b;] instead", :getindex) @@ -654,3 +653,34 @@ function getindex{T<:Union{Char,Number}}(::Type{T}, r1::Range, rs::Range...) end return a end + +function require(mod::AbstractString) + depwarn("`require` is deprecated, use `using` or `import` instead", :require) + require(symbol(require_filename(mod))) +end +function require(f::AbstractString, fs::AbstractString...) + require(f) + for fn in fs + require(fn) + end +end +export require +function require_filename(name::AbstractString) + # This function can be deleted when the deprecation for `require` + # is deleted. + # While we could also strip off the absolute path, the user may be + # deliberately directing to a different file than what got + # cached. So this takes a conservative approach. + if endswith(name, ".jl") + tmp = name[1:end-3] + for prefix in LOAD_CACHE_PATH + path = joinpath(prefix, tmp*".ji") + if isfile(path) + return tmp + end + end + end + name +end +const reload = require +export reload diff --git a/base/exports.jl b/base/exports.jl index 1bee26066a743..4dfe4b6d5088f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1096,8 +1096,6 @@ export evalfile, include, include_string, - reload, - require, # RTS internals finalizer, diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 3f4e70aeeed54..af7282e07187d 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -385,7 +385,6 @@ function workspace() Expr(:toplevel, :(const Base = $(Expr(:quote, b))), :(const LastMain = $(Expr(:quote, last))))) - empty!(package_list) empty!(package_locks) nothing end diff --git a/base/loading.jl b/base/loading.jl index b6eb34c9e05e3..45467c526d861 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# require +# Base.require is the implementation for the `import` statement function find_in_path(name::AbstractString) isabspath(name) && return name @@ -23,8 +23,8 @@ function find_in_path(name::AbstractString) return nothing end -find_in_node1_path(name) = myid()==1 ? - find_in_path(name) : remotecall_fetch(1, find_in_path, name) +find_in_node_path(name, node::Int=1) = myid() == node ? + find_in_path(name) : remotecall_fetch(node, find_in_path, name) function find_source_file(file) (isabspath(file) || isfile(file)) && return file @@ -34,62 +34,114 @@ function find_source_file(file) isfile(file2) ? file2 : nothing end -# Store list of files and their load time -package_list = Dict{ByteString,Float64}() -# to synchronize multiple tasks trying to require something -package_locks = Dict{ByteString,Any}() -require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs require(x); end) - -# only broadcast top-level (not nested) requires and reloads -toplevel_load = true +function find_in_cache_path(mod::Symbol) + name = string(mod) + for prefix in LOAD_CACHE_PATH + path = joinpath(prefix, name*".ji") + if isfile(path) + produce(path) + end + end + nothing +end -function require(name::AbstractString) - path = find_in_node1_path(name) - path == nothing && throw(ArgumentError("$name not found in path")) +function _include_from_serialized(content::Vector{UInt8}) + m = ccall(:jl_restore_incremental_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content)) + return m != 0 +end - if myid() == 1 && toplevel_load - refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ] - _require(path) - for r in refs; wait(r); end +function _require_from_serialized(node::Int, path_to_try::ByteString, toplevel_load::Bool) + if toplevel_load && myid() == 1 && nprocs() > 1 + # broadcast top-level import/using from node 1 (only) + if node == myid() + content = open(readbytes, path_to_try) + else + content = remotecall_fetch(node, open, readbytes, path_to_try) + end + if _include_from_serialized(content) + others = filter(x -> x != myid(), procs()) + refs = Any[ @spawnat p _include_from_serialized(content) for p in others] + for (id, ref) in zip(others, refs) + if !fetch(ref) + warn("node state is inconsistent: node $id failed to load cache from $path_to_try") + end + end + return true + end + elseif node == myid() + if ccall(:jl_restore_incremental, UInt, (Ptr{Uint8},), path_to_try) != 0 + return true + end else - _require(path) + content = remotecall_fetch(node, open, readbytes, path_to_try) + if _include_from_serialized(content) + return true + end end - nothing + # otherwise, continue search + return false end -function _require(path) - global toplevel_load - if haskey(package_list,path) - loaded, c = package_locks[path] - !loaded && wait(c) - else - last = toplevel_load - toplevel_load = false - try - reload_path(path) - finally - toplevel_load = last +function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) + name = string(mod) + finder = @spawnat node @task find_in_cache_path(mod) # TODO: switch this to an explicit Channel + while true + path_to_try = remotecall_fetch(node, finder->consume(fetch(finder)), finder) + path_to_try === nothing && return false + if _require_from_serialized(node, path_to_try, toplevel_load) + return true + else + warn("deserialization checks failed while attempting to load cache from $path_to_try") end end end -function reload(name::AbstractString) +# to synchronize multiple tasks trying to import/using something +package_locks = Dict{Symbol,Condition}() +package_loaded = Set{Symbol}() + +# require always works in Main scope and loads files from node 1 +toplevel_load = true +function require(mod::Symbol) global toplevel_load - path = find_in_node1_path(name) - path == nothing && throw(ArgumentError("$name not found in path")) - refs = nothing - if myid() == 1 && toplevel_load - refs = Any[ @spawnat p reload_path(path) for p in filter(x->x!=1, procs()) ] + loading = get(package_locks, mod, false) + if loading !== false + # load already in progress for this module + wait(loading) + return end + package_locks[mod] = Condition() + last = toplevel_load - toplevel_load = false try - reload_path(path) + toplevel_load = false + if _require_from_serialized(1, mod, last) + return true + end + if JLOptions().incremental != 0 + # spawn off a new incremental compile task from node 1 for recursive `require` calls + cachefile = compile(mod) + if !_require_from_serialized(1, cachefile, last) + warn("require failed to create a precompiled cache file") + end + return + end + + name = string(mod) + path = find_in_node_path(name, 1) + path === nothing && throw(ArgumentError("$name not found in path")) + if last && myid() == 1 && nprocs() > 1 + # broadcast top-level import/using from node 1 (only) + content = open(readall, path) + refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ] + for r in refs; wait(r); end + else + eval(Main, :(Base.include_from_node1($path))) + end finally toplevel_load = last - end - if refs !== nothing - for r in refs; wait(r); end + loading = pop!(package_locks, mod) + notify(loading, all=true) end nothing end @@ -145,38 +197,62 @@ function include_from_node1(path::AbstractString) result end -function reload_path(path::AbstractString) - had = haskey(package_list, path) - if !had - package_locks[path] = (false, Condition()) - end - package_list[path] = time() - tls = task_local_storage() - prev = pop!(tls, :SOURCE_PATH, nothing) - try - eval(Main, :(Base.include_from_node1($path))) - catch e - had || delete!(package_list, path) - rethrow(e) - finally - if prev != nothing - tls[:SOURCE_PATH] = prev - end - end - reloaded, c = package_locks[path] - if !reloaded - package_locks[path] = (true, c) - notify(c, all=true) - end - nothing -end - function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) return eval(Module(:__anon__), Expr(:toplevel, :(const ARGS = $args), - :(eval(x) = Core.eval(__anon__,x)), - :(eval(m,x) = Core.eval(m,x)), - :(include($path)))) + :(eval(x) = Main.Core.eval(__anon__,x)), + :(eval(m,x) = Main.Core.eval(m,x)), + :(Main.Base.include($path)))) end evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...]) + +function create_expr_cache(input::AbstractString, output::AbstractString) + code_object = """ + while !eof(STDIN) + eval(Main, deserialize(STDIN)) + end + """ + io, pobj = open(detach(setenv(`$(julia_cmd()) + --output-ji $output --output-incremental=yes + --startup-file=no --history-file=no + --eval $code_object`, + ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w", STDOUT) + serialize(io, quote + empty!(Base.LOAD_PATH) + append!(Base.LOAD_PATH, $LOAD_PATH) + empty!(Base.LOAD_CACHE_PATH) + append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH) + empty!(Base.DL_LOAD_PATH) + append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH) + end) + source = source_path(nothing) + if source !== nothing + serialize(io, quote + task_local_storage()[:SOURCE_PATH] = $(source) + end) + end + serialize(io, :(Base.include($(abspath(input))))) + if source !== nothing + serialize(io, quote + delete!(task_local_storage(), :SOURCE_PATH) + end) + end + close(io) + wait(pobj) + return pobj +end + +function compile(mod::Symbol) + myid() == 1 || error("can only compile from node 1") + name = string(mod) + path = find_in_path(name) + path === nothing && throw(ArgumentError("$name not found in path")) + cachepath = LOAD_CACHE_PATH[1] + if !isdir(cachepath) + mkpath(cachepath) + end + cachefile = abspath(cachepath, name*".ji") + create_expr_cache(path, cachefile) + return cachefile +end diff --git a/base/options.jl b/base/options.jl index ca1810f4c813b..fd4f7c3211b38 100644 --- a/base/options.jl +++ b/base/options.jl @@ -32,6 +32,7 @@ immutable JLOptions outputbc::Ptr{UInt8} outputo::Ptr{UInt8} outputji::Ptr{UInt8} + incremental::Int8 end JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions)) diff --git a/base/precompile.jl b/base/precompile.jl index f0b913e297daf..fdfa20deffe4d 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -162,7 +162,6 @@ precompile(Base._deleteat_beg!, (Array{UInt8, 1}, Int, Int)) precompile(Base._deleteat_end!, (Array{UInt8, 1}, Int, Int)) precompile(Base._growat_beg!, (Array{UInt8, 1}, Int, Int)) precompile(Base._growat_end!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._require, (ASCIIString,)) precompile(Base._setindex!, (Base.Dict{Symbol, Any}, Base.LineEdit.Prompt, Symbol, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Base.LineEdit.PromptState, Base.LineEdit.Prompt, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Bool, WeakRef, Int)) @@ -253,6 +252,7 @@ precompile(Base.in, (Char, ASCIIString)) precompile(Base.in, (Int, Base.UnitRange{Int})) precompile(Base.in, (UInt8, Base.KeyIterator{Dict{UInt8, Any}})) precompile(Base.include_from_node1, (ASCIIString,)) +precompile(Base.include_from_node1, (UTF8String,)) precompile(Base.init_stdio, (Ptr{Void},)) precompile(Base.input_color, ()) precompile(Base.insert!, (Array{Any,1}, Int, Base.GlobalRef)) @@ -351,10 +351,9 @@ precompile(Base.readuntil, (IOBuffer, UInt8)) precompile(Base.rehash!, (Dict{Any,Any}, Int)) precompile(Base.rehash!, (Dict{UInt8, Any}, Int)) precompile(Base.reinit_stdio, ()) -precompile(Base.reload_path, (ASCIIString,)) precompile(Base.repeat, (ASCIIString, Int)) precompile(Base.repl_cmd, (Cmd,)) -precompile(Base.require, (ASCIIString,)) +precompile(Base.require, (Symbol,)) precompile(Base.rr2id, (RemoteRef,)) precompile(Base.rsearch, (ASCIIString, Char)) precompile(Base.rstrip, (ASCIIString,)) diff --git a/base/sysimg.jl b/base/sysimg.jl index 71fea54c4a0de..afe363fab0248 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -1,7 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license import Core.Intrinsics.ccall -ccall(:jl_new_main_module, Any, ()) baremodule Base @@ -314,9 +313,8 @@ function __init__() init_parallel() end -include("precompile.jl") - include = include_from_node1 +include("precompile.jl") end # baremodule Base diff --git a/doc/helpdb.jl b/doc/helpdb.jl index c7bf85f074ac5..fc986af5dcbe3 100644 --- a/doc/helpdb.jl +++ b/doc/helpdb.jl @@ -1308,9 +1308,15 @@ Any[ "), -("Base","require","require(file::AbstractString...) +("Base","require","require(module::Symbol) - Load source files once, in the context of the \"Main\" module, on + This function is part of the implementation of \"using\" / + \"import\", if a module is not already defined in \"Main\". It can + also be called directly to force reloading a module, regardless of + whether it has been loaded before (for exmple, when interactively + developing libraries). + + Loads a source files, in the context of the \"Main\" module, on every active node, searching standard locations for files. \"require\" is considered a top-level operation, so it sets the current \"include\" path but does not use it to search for files @@ -1324,11 +1330,14 @@ Any[ "), -("Base","reload","reload(file::AbstractString) +("Base","compile","compile(module::String) - Like \"require\", except forces loading of files regardless of - whether they have been loaded before. Typically used when - interactively developing libraries. + Creates a precompiled cache file for module (see help for + \"require\") and all of its dependencies. This can be used to + reduce package load times. Cache files are stored in + LOAD_CACHE_PATH[1], which defaults to *~/.julia/lib/VERSION*. See + the manual section *Module initialization and precompilation* + (under *Modules*) for important notes. "), @@ -1345,7 +1354,7 @@ Any[ "), -("Base","include_string","include_string(code::AbstractString) +("Base","include_string","include_string(code::AbstractString[, filename]) Like \"include\", except reads code from the given string rather than from a file. Since there is no file path involved, no path @@ -5701,18 +5710,26 @@ Millisecond(v) "), -("Base","readbytes!","readbytes!(stream, b::Vector{UInt8}, nb=length(b)) +("Base","readbytes!","readbytes!(stream, b::Vector{UInt8}, nb=length(b); all=true) Read at most \"nb\" bytes from the stream into \"b\", returning the number of bytes read (increasing the size of \"b\" as needed). + See \"readbytes\" for a description of the \"all\" option. + "), -("Base","readbytes","readbytes(stream, nb=typemax(Int)) +("Base","readbytes","readbytes(stream, nb=typemax(Int); all=true) Read at most \"nb\" bytes from the stream, returning a \"Vector{UInt8}\" of the bytes read. + If \"all\" is true (the default), this function will block + repeatedly trying to read all requested bytes, until an error or + end-of-file occurs. If \"all\" is false, at most one \"read\" call + is performed, and the amount of data returned is device-dependent. + Note that not all stream types support the \"all\" option. + "), ("Base","position","position(s) @@ -5920,8 +5937,8 @@ Millisecond(v) ("Base","countlines","countlines(io[, eol::Char]) - Read io until the end of the stream/file and count the number of - non-empty lines. To specify a file pass the filename as the first + Read \"io\" until the end of the stream/file and count the number + of lines. To specify a file pass the filename as the first argument. EOL markers other than '\\n' are supported by passing them as the second argument. @@ -6387,31 +6404,55 @@ popdisplay(d::Display) "), -("Base","mmap_array","mmap_array(type, dims, stream[, offset]) +("Base","Mmap","Mmap.Anonymous(name, readonly, create) + + Create an \"IO\"-like object for creating zeroed-out mmapped-memory + that is not tied to a file for use in \"Mmap.mmap\". Used by + \"SharedArray\" for creating shared memory arrays. + +"), + +("Base","Mmap","Mmap.mmap(io::Union(IOStream,AbstractString,Mmap.AnonymousMmap)[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) +Mmap.mmap(type::Type{Array{T, N}}, dims) Create an \"Array\" whose values are linked to a file, using memory-mapping. This provides a convenient way of working with data too large to fit in the computer's memory. - The type determines how the bytes of the array are interpreted. - Note that the file must be stored in binary format, and no format - conversions are possible (this is a limitation of operating - systems, not Julia). + The type is an \"Array{T,N}\" with a bits-type element of \"T\" and + dimension \"N\" that determines how the bytes of the array are + interpreted. Note that the file must be stored in binary format, + and no format conversions are possible (this is a limitation of + operating systems, not Julia). + + \"dims\" is a tuple or single \"Integer\" specifying the size or + length of the array. - \"dims\" is a tuple specifying the size of the array. + The file is passed via the stream argument, either as an open + \"IOStream\" or filename string. When you initialize the stream, + use \"\"r\"\" for a \"read-only\" array, and \"\"w+\"\" to create a + new array used to write values to disk. - The file is passed via the stream argument. When you initialize - the stream, use \"\"r\"\" for a \"read-only\" array, and \"\"w+\"\" - to create a new array used to write values to disk. + If no \"type\" argument is specified, the default is + \"Vector{UInt8}\". Optionally, you can specify an offset (in bytes) if, for example, you want to skip over a header in the file. The default value for - the offset is the current stream position. + the offset is the current stream position for an \"IOStream\". + + The \"grow\" keyword argument specifies whether the disk file + should be grown to accomodate the requested size of array (if the + total file size is < requested array size). Write privileges are + required to grow the file. + + The \"shared\" keyword argument specifies whether the resulting + \"Array\" and changes made to it will be visible to other processes + mapping the same file. For example, the following code: # Create a file for mmapping - # (you could alternatively use mmap_array to do this step, too) + # (you could alternatively use mmap to do this step, too) A = rand(1:20, 5, 30) s = open(\"/tmp/mmap.bin\", \"w+\") # We'll write the dimensions of the array as the first two Ints in the file @@ -6425,7 +6466,7 @@ popdisplay(d::Display) s = open(\"/tmp/mmap.bin\") # default is read-only m = read(s, Int) n = read(s, Int) - A2 = mmap_array(Int, (m,n), s) + A2 = Mmap.mmap(s, Matrix{Int}, (m,n)) creates a \"m\"-by-\"n\" \"Matrix{Int}\", linked to the file associated with stream \"s\". @@ -6437,22 +6478,21 @@ popdisplay(d::Display) "), -("Base","mmap_bitarray","mmap_bitarray([type], dims, stream[, offset]) +("Base","Mmap","Mmap.mmap(io, BitArray[, dims, offset]) Create a \"BitArray\" whose values are linked to a file, using memory-mapping; it has the same purpose, works in the same way, and - has the same arguments, as \"mmap_array()\", but the byte - representation is different. The \"type\" parameter is optional, - and must be \"Bool\" if given. + has the same arguments, as \"mmap()\", but the byte representation + is different. - **Example**: \"B = mmap_bitarray((25,30000), s)\" + **Example**: \"B = Mmap.mmap(s, BitArray, (25,30000))\" This would create a 25-by-30000 \"BitArray\", linked to the file associated with stream \"s\". "), -("Base","msync","msync(array) +("Base","Mmap","Mmap.sync!(array) Forces synchronization between the in-memory version of a memory- mapped \"Array\" or \"BitArray\" and the on-disk version. @@ -6794,12 +6834,18 @@ popdisplay(d::Display) \"RTLD_FIRST\". These are converted to the corresponding flags of the POSIX (and/or GNU libc and/or MacOS) dlopen command, if possible, or are ignored if the specified functionality is not - available on the current platform. The default is - \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL\". An important usage of - these flags, on POSIX platforms, is to specify - \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" in order for the library's - symbols to be available for usage in other shared libraries, in - situations where there are dependencies between shared libraries. + available on the current platform. The default flags are platform + specific. On MacOS the default \"dlopen\" flags are + \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" while on other platforms + the defaults are \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL\". An + important usage of these flags is to specify non default behavior + for when the dynamic library loader binds library references to + exported symbols and if the bound references are put into process + local or global scope. For instance + \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" allows the library's + symbols to be available for usage in other shared libraries, + addressing situations where there are dependencies between shared + libraries. "), @@ -6991,15 +7037,15 @@ popdisplay(d::Display) \"Complex{Float64}\" the return type is \"UmfpackLU\". Some examples are shown in the table below. - +-------------------------+---------------------------+----------------------------------------------+ - | Type of input \\\"A\\\" | Type of output \\\"F\\\" | Relationship between \\\"F\\\" and \\\"A\\\" | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"Matrix()\\\" | \\\"LU\\\" | \\\"F[:L]*F[:U] == A[F[:p], :]\\\" | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"Tridiagonal()\\\" | \\\"LU{T,Tridiagonal{T}}\\\" | N/A | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"SparseMatrixCSC()\\\" | \\\"UmfpackLU\\\" | \\\"F[:L]*F[:U] == F[:Rs] .* A[F[:p], F[:q]]\\\" | - +-------------------------+---------------------------+----------------------------------------------+ + +-------------------------+---------------------------+------------------------------------------------+ + | Type of input \\\"A\\\" | Type of output \\\"F\\\" | Relationship between \\\"F\\\" and \\\"A\\\" | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"Matrix()\\\" | \\\"LU\\\" | \\\"F[:L]*F[:U] == A[F[:p], :]\\\" | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"Tridiagonal()\\\" | \\\"LU{T,Tridiagonal{T}}\\\" | N/A | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"SparseMatrixCSC()\\\" | \\\"UmfpackLU\\\" | \\\"F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]\\\" | + +-------------------------+---------------------------+------------------------------------------------+ The individual components of the factorization \"F\" can be accessed by indexing: @@ -10278,12 +10324,12 @@ popdisplay(d::Display) "), -("Base","median","median(v) +("Base","median","median(v[, region]) - Compute the median of a vector \"v\". \"NaN\" is returned if the - data contains any \"NaN\" values. For applications requiring the - handling of missing data, the \"DataArrays\" package is - recommended. + Compute the median of whole array \"v\", or optionally along the + dimensions in \"region\". \"NaN\" is returned if the data contains + any \"NaN\" values. For applications requiring the handling of + missing data, the \"DataArrays\" package is recommended. "), @@ -10562,7 +10608,7 @@ popdisplay(d::Display) the transform has conjugate symmetry in order to save roughly half the computational time and storage costs compared with \"fft()\". If \"A\" has size \"(n_1, ..., n_d)\", the result has size - \"(floor(n_1/2)+1, ..., n_d)\". + \"(div(n_1,2)+1, ..., n_d)\". The optional \"dims\" argument specifies an iterable subset of one or more dimensions of \"A\" to transform, similar to \"fft()\". @@ -10580,10 +10626,11 @@ popdisplay(d::Display) transform, defaulting to \"1:ndims(A)\". \"d\" is the length of the transformed real array along the - \"dims[1]\" dimension, which must satisfy \"d == - floor(size(A,dims[1])/2)+1\". (This parameter cannot be inferred - from \"size(A)\" due to the possibility of rounding by the - \"floor\" function here.) + \"dims[1]\" dimension, which must satisfy \"div(d,2)+1 == + size(A,dims[1])\". (This parameter cannot be inferred from + \"size(A)\" since both \"2*size(A,dims[1])-2\" as well as + \"2*size(A,dims[1])-1\" are valid sizes for the transformed real + array.) "), @@ -13235,8 +13282,9 @@ golden ("Base.Test","@test_throws","@test_throws(extype, ex) Test that the expression \"ex\" throws an exception of type - \"extype\" and calls the current handler to handle the result. - The default handler returns the exception if it is of the expected type. + \"extype\" and calls the current handler to handle the result. The + default handler returns the exception if it is of the expected + type. "), diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 09e2acb909653..aa47052141b31 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -254,13 +254,21 @@ Module initialization and precompilation ---------------------------------------- Large modules can take several second to load because executing all of -the statements in a module often involves compiling a large amount of -code. However, Julia is progressively gaining more ability to cache -the parsed and compiled binary image of a package. Currently, this -requires one to recompile Julia after modifying the file -``base/userimg.jl`` to require the desired modules, but in a future -version of Julia the module caching will be simpler and more -automated. In order to make your module work with precompilation, +the statements in a module often involves compiling a large amount of code. +Julia provides the ability to create precompiled versions of modules +to reduce this time. + +There are two mechanisms that can achieve this: +incremental compile and custom system image. + +To create a custom system image that can be used to start julia with the -J option, +recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. + +To create an incremental precompiled module file, +call ``Base.compile(modulename::Symbol)``. +The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. + +In order to make your module work with precompilation, however, you may need to change your module to explicitly separate any initialization steps that must occur at *runtime* from steps that can occur at *compile time*. For this purpose, Julia allows you to define @@ -293,18 +301,20 @@ in your module:: global const foo_data_ptr = ccall((:foo_data,:libfoo), Ptr{Void}, ()) end -(Notice that it is perfectly possible to define a global constant inside +Notice that it is perfectly possible to define a global inside a function like ``__init__``; this is one of the advantages of using a -dynamic language.) Obviously, any other constant in your module that -depends on ``foo_data_ptr`` would also have to be initialized in ``__init__``. +dynamic language. +Obviously, any other globals in your module that depends on ``foo_data_ptr`` +would also have to be initialized in ``__init__``. Constants involving most Julia objects that are not produced by ``ccall`` do not need to be placed in ``__init__``: their definitions -can be precompiled and loaded from the cached module image. (This -includes complicated heap-allocated objects like arrays.) However, -any routine that returns a raw pointer value must be called at runtime -for precompilation to work. This includes the Julia functions -``cfunction`` and ``pointer``. +can be precompiled and loaded from the cached module image. +This includes complicated heap-allocated objects like arrays. +However, any routine that returns a raw pointer value must be called +at runtime for precompilation to work +(Ptr objects will turn into null pointers unless they are hidden inside an isbits object). +This includes the return values of the Julia functions ``cfunction`` and ``pointer``. Dictionary and set types, or in general anything that depends on the output of a ``hash(key)`` method, are a trickier case. In the common @@ -314,9 +324,80 @@ they are safe to precompile. However, for a few other key types, such as ``Function`` or ``DataType`` and generic user-defined types where you haven't defined a ``hash`` method, the fallback ``hash`` method depends on the memory address of the object (via its ``object_id``) -and hence may change from run to run. If you have one of these key -types, or if you aren't sure, to be safe you can initialize dictionary -and set globals from within your ``__init__`` function. -Alternatively, you can use the ``ObjectIdDict`` dictionary type, which -is specially handled by precompilation so that it is safe to +and hence may change from run to run. +If you have one of these key types, or if you aren't sure, +to be safe you can initialize this dictionary from within your +``__init__`` function. +Alternatively, you can use the ``ObjectIdDict`` dictionary type, +which is specially handled by precompilation so that it is safe to initialize at compile-time. + +When using precompilation, it is important to keep a clear sense of the +distinction between the compilation phase and the execution phase. +In this mode, it will often be much more clearly apparent that +Julia is a compiler which allows execution of arbitrary Julia code, +not a standalone interpreter that also generates compiled code. + +Other known potential failure scenarios include: + +1. Global counters (for example, for uniquely identify objects) + Consider the following code snippet:: + + type UniquedById + myid::Int + let counter = 0 + UniquedById() = new(counter += 1) + end + end + + while the intent of this code was to give every instance a unique id, + the counter value is recorded at the end of compilation. + All subsequent usages of this incrementally compiled module + will start from that same counter value. + + One alternative is to store both ``current_module()`` and the current ``counter`` value, + however, it may be better to redesign the code to not depend on this global state. + +2. Associative collections (such as ``Dict`` and ``Set``) need to be re-hashed in ``__init__``. + (In the future, a mechanism may be provided to register an initializer function.) + +3. Depending on compile-time side-effects persisting through load-time. + Example include: + modifying arrays or other variables in other Julia modules; + maintaining handles to open files or devices; + storing pointers to other system resources (including memory); + +4. Creating accidental "copies" of global state from another module, + by referencing it directly instead of via its lookup path. + For example, (in global scope):: + #mystdout = Base.STDOUT #= will not work correctly, since this will copy Base.STDOUT into this module =# + # instead use accessor functions: + getstdout() = Base.STDOUT #= best option =# + # or move the assignment into the runtime: + __init__() = global mystdout = Base.STDOUT #= also works =# + +Several additional restrictions are placed on the operations that can be done while compiling code +to help the user avoid other wrong-behavior situations: + +1. Calling ``eval`` to cause a side-effect in another module. + This will also cause a warning to be emitted when the incremental compile flag is set. + +2. ``global const`` statements from local scope after ``__init__()`` has been started (see issue #12010 for plans to add an error for this) + +3. Replacing a module (or calling ``workspace()`` is a runtime error while doing an incremental compile. + +A few other points to be aware of: + +1. No code reload / cache invalidation is performed after changes are made to the source files themselves, + (including by ``Pkg.update``), and no cleanup is done after ``Pkg.rm`` + +2. The memory sharing behavior of a reshaped array is disregarded by precompilation (each view gets its own copy) + +3. Expecting the filesystem to be unchanged between compile-time and runtime + e.g. ``@__FILE__``/``source_path()`` to find resources at runtime, + or the BinDeps ``@checked_lib`` macro. Sometimes this is unavoidable. + However, when possible, it can be good practice to copy resources + into the module at compile-time so they won't need to be found at runtime. + +4. WeakRef objects and finalizers are not captured by currently handled by the serializer + (this will be fixed in an upcoming release). diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index e7ead054d2064..5ce55500c1167 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -78,21 +78,23 @@ Getting Around Return a string with the contents of the operating system clipboard ("paste"). -.. function:: require(file::AbstractString...) +.. function:: require(module::Symbol) - Load source files once, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. + This function is part of the implementation of ``using`` / ``import``, if a module is not already defined in ``Main``. It can also be called directly to force reloading a module, regardless of whether it has been loaded before (for exmple, when interactively developing libraries). + + Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. When searching for files, ``require`` first looks in the current working directory, then looks for package code under ``Pkg.dir()``, then tries paths in the global array ``LOAD_PATH``. -.. function:: reload(file::AbstractString) +.. function:: compile(module::String) - Like ``require``, except forces loading of files regardless of whether they have been loaded before. Typically used when interactively developing libraries. + Creates a precompiled cache file for module (see help for ``require``) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in LOAD_CACHE_PATH[1], which defaults to `~/.julia/lib/VERSION`. See the manual section `Module initialization and precompilation` (under `Modules`) for important notes. .. function:: include(path::AbstractString) Evaluate the contents of a source file in the current context. During including, a task-local include path is set to the directory containing the file. Nested calls to ``include`` will search relative to that path. All paths refer to files on node 1 when running in parallel, and files will be fetched from node 1. This function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. -.. function:: include_string(code::AbstractString) +.. function:: include_string(code::AbstractString, [filename]) Like ``include``, except reads code from the given string rather than from a file. Since there is no file path involved, no path processing or fetching from node 1 is done. diff --git a/src/dump.c b/src/dump.c index 1f08520b28daf..6586a95585bf0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -25,6 +25,9 @@ extern "C" { #endif +// TODO: put WeakRefs on the weak_refs list during deserialization +// TODO: handle finalizers + // hash of definitions for predefined tagged object static htable_t ser_tag; // array of definitions for the predefined tagged object types @@ -43,19 +46,24 @@ int backref_table_numel; static arraylist_t backref_list; // list of (jl_value_t **loc, size_t pos) entries -// for anything that was flagged by the serializer for later +// for anything that was flagged by the deserializer for later // type-rewriting of some sort static arraylist_t flagref_list; // list of (size_t pos, (void *f)(jl_value_t*)) entries // for the serializer to mark values in need of rework by function f +// during deserialization later static arraylist_t reinit_list; // list of any methtable objects that were deserialized in MODE_MODULE // and need to be rehashed after assigning the uid fields to types -// (only used in MODE_MODULE and MODE_MODULE_LAMBDAS) +// (only used in MODE_MODULE and MODE_MODULE_POSTWORK) static arraylist_t methtable_list; +// list of stuff that is being serialized +// (only used by the incremental serializer in MODE_MODULE) +static jl_array_t *serializer_worklist; + // hash of definitions for predefined function pointers static htable_t fptr_to_id; // array of definitions for the predefined function pointers @@ -81,7 +89,7 @@ static const ptrint_t LongSvec_tag = 24; static const ptrint_t LongExpr_tag = 25; static const ptrint_t LiteralVal_tag = 26; static const ptrint_t SmallInt64_tag = 27; -static const ptrint_t UNUSED_tag = 28; +static const ptrint_t SmallDataType_tag= 28; static const ptrint_t Int32_tag = 29; static const ptrint_t Array1d_tag = 30; static const ptrint_t Singleton_tag = 31; @@ -110,7 +118,7 @@ typedef enum _DUMP_MODES { // restoring a single module from disk for integration // into the currently running system image / environment MODE_MODULE, // first-stage (pre type-uid assignment) - MODE_MODULE_LAMBDAS, // second-stage (post type-uid assignment) + MODE_MODULE_POSTWORK, // second-stage (post type-uid assignment) } DUMP_MODES; static DUMP_MODES mode = (DUMP_MODES) 0; @@ -212,7 +220,7 @@ static int jl_load_sysimg_so() if (jl_sysimg_handle == 0) return -1; - int imaging_mode = jl_generating_output(); + int imaging_mode = jl_generating_output() && !jl_options.incremental; // in --build mode only use sysimg data, not precompiled native code if (!imaging_mode && jl_options.use_precompiled==JL_OPTIONS_USE_PRECOMPILED_YES) { sysimg_gvars = (jl_value_t***)jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars"); @@ -282,9 +290,10 @@ static void jl_serialize_globalvals(ios_t *s) for(i=0; i < len; i+=2) { void *offs = p[i+1]; if (offs != HT_NOTFOUND) { + uintptr_t pos = offs - HT_NOTFOUND - 1; int32_t gv = jl_get_llvm_gv((jl_value_t*)p[i]); if (gv != 0) { - write_int32(s, (int)(intptr_t)offs); + write_int32(s, pos); write_int32(s, gv); } } @@ -392,15 +401,37 @@ static void jl_serialize_fptr(ios_t *s, void *fptr) write_uint16(s, *(ptrint_t*)pbp); } +static int module_in_worklist(jl_module_t *mod) { + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *workmod = (jl_module_t*)jl_cellref(serializer_worklist, i); + if (jl_is_module(workmod) && jl_is_submodule(mod, workmod)) + return 1; + } + return 0; +} + static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) { int tag = 0; - if (mode == MODE_MODULE_LAMBDAS) { - if (dt->uid != 0) - tag = 6; // must use apply_type + if (mode == MODE_MODULE_POSTWORK) { + if (dt->uid != 0) { + if (dt->name->primary == (jl_value_t*)dt) + tag = 6; // primary type + else + tag = 7; // must use apply_type + } } else if (mode == MODE_MODULE) { - int internal = jl_is_submodule(dt->name->module, jl_current_module); + int internal = module_in_worklist(dt->name->module); + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *mod = (jl_module_t*)jl_cellref(serializer_worklist, i); + if (jl_is_module(mod) && jl_is_submodule(dt->name->module, mod)) { + internal = 1; + break; + } + } if (!internal && dt->name->primary == (jl_value_t*)dt) { tag = 6; // external primary type } @@ -415,7 +446,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) // also flag this in the backref table as special uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, dt); assert(*bp != (uptrint_t)HT_NOTFOUND); - *bp |= 1; + *bp |= 1; assert(((uptrint_t)HT_NOTFOUND)|1); } } else if (dt == jl_int32_type) @@ -424,7 +455,8 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) tag = 3; else if (dt == jl_int64_type) tag = 4; - writetag(s, (jl_value_t*)jl_datatype_type); + writetag(s, (jl_value_t*)SmallDataType_tag); + write_uint8(s, 0); // virtual size jl_serialize_value(s, (jl_value_t*)jl_datatype_type); write_uint8(s, tag); if (tag == 6) { @@ -443,7 +475,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) write_uint8(s, dt->abstract | (dt->mutabl<<1) | (dt->pointerfree<<2) | (has_instance<<3)); if (!dt->abstract) { write_uint16(s, dt->ninitialized); - if (mode != MODE_MODULE && mode != MODE_MODULE_LAMBDAS) { + if (mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK) { write_int32(s, dt->uid); } } @@ -466,18 +498,20 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) writetag(s, jl_module_type); jl_serialize_value(s, m->name); int ref_only = 0; - if (mode == MODE_MODULE_LAMBDAS) { - assert(!jl_is_submodule(m, jl_current_module)); + if (mode == MODE_MODULE_POSTWORK) { + assert(!module_in_worklist(m)); ref_only = 1; } if (mode == MODE_MODULE) { - if (!jl_is_submodule(m, jl_current_module)) + if (!module_in_worklist(m)) ref_only = 1; write_int8(s, ref_only); } jl_serialize_value(s, m->parent); - if (ref_only) + if (ref_only) { + assert(m->parent != m); return; + } size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { @@ -570,24 +604,34 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) else { bp = ptrhash_bp(&backref_table, v); if (*bp != HT_NOTFOUND) { + uintptr_t pos = *bp - HT_NOTFOUND - 1; if ((uptrint_t)*bp < 65536) { write_uint8(s, ShortBackRef_tag); - write_uint16(s, (uptrint_t)*bp); + write_uint16(s, pos); } else { write_uint8(s, BackRef_tag); - write_int32(s, (uptrint_t)*bp); + write_int32(s, pos); } return; } ptrint_t pos = backref_table_numel++; - if (mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) - pos <<= 1; - ptrhash_put(&backref_table, v, (void*)pos); if (jl_typeof(v) == jl_idtable_type) { + // will need to rehash this, later (after types are fully constructed) arraylist_push(&reinit_list, (void*)pos); arraylist_push(&reinit_list, (void*)1); } + if (mode == MODE_MODULE && jl_is_module(v)) { + jl_module_t *m = (jl_module_t*)v; + if (module_in_worklist(m) && !module_in_worklist(m->parent)) { + // will need to reinsert this into parent bindings, later (in case of any errors during reinsert) + arraylist_push(&reinit_list, (void*)pos); + arraylist_push(&reinit_list, (void*)2); + } + } + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) + pos <<= 1; + ptrhash_put(&backref_table, v, HT_NOTFOUND + pos + 1); } size_t i; @@ -639,9 +683,9 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) write_uint16(s, ar->ndims); write_uint16(s, (ar->ptrarray<<15) | (ar->elsize & 0x7fff)); } - jl_serialize_value(s, jl_typeof(ar)); for (i=0; i < ar->ndims; i++) jl_serialize_value(s, jl_box_long(jl_array_dim(ar,i))); + jl_serialize_value(s, jl_typeof(ar)); if (!ar->ptrarray) { size_t tot = jl_array_len(ar) * ar->elsize; ios_write(s, (char*)jl_array_data(ar), tot); @@ -651,9 +695,6 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, jl_cellref(v, i)); } } - if (mode == MODE_MODULE) { - jl_serialize_value(s, jl_typeof(ar)); - } } else if (jl_is_expr(v)) { jl_expr_t *e = (jl_expr_t*)v; @@ -684,6 +725,18 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else if (jl_is_function(v)) { writetag(s, jl_function_type); + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) { + if (jl_is_gf(v)) { + jl_methtable_t *mt = jl_gf_mtable(v); + if (mt->module && !module_in_worklist(mt->module)) { + write_int8(s, 1); + jl_serialize_value(s, (jl_value_t*)mt->module); + jl_serialize_value(s, (jl_value_t*)mt->name); + return; + } + } + write_int8(s, 0); + } jl_function_t *f = (jl_function_t*)v; jl_serialize_value(s, (jl_value_t*)f->linfo); jl_serialize_value(s, f->env); @@ -756,14 +809,27 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { if (v == t->instance) { - assert(mode != MODE_MODULE_LAMBDAS); + if (mode == MODE_MODULE) { + // also flag this in the backref table as special + uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, v); + assert(*bp != (uptrint_t)HT_NOTFOUND); + *bp |= 1; assert(((uptrint_t)HT_NOTFOUND)|1); + } writetag(s, (jl_value_t*)Singleton_tag); + jl_serialize_value(s, t); return; } - writetag(s, (jl_value_t*)jl_datatype_type); + if (t->size <= 255) { + writetag(s, (jl_value_t*)SmallDataType_tag); + write_uint8(s, t->size); + } + else { + writetag(s, (jl_value_t*)jl_datatype_type); + write_int32(s, t->size); + } jl_serialize_value(s, t); - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && t == jl_typename_type) { - if (jl_is_submodule(((jl_typename_t*)v)->module, jl_current_module)) { + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && t == jl_typename_type) { + if (module_in_worklist(((jl_typename_t*)v)->module)) { write_uint8(s, 0); } else { @@ -788,73 +854,60 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { for(size_t i=0; i < nf; i++) { - jl_serialize_value(s, jl_get_nth_field(v, i)); + if (t->fields[i].size > 0) { + jl_serialize_value(s, jl_get_nth_field(v, i)); + } } } - if (mode == MODE_MODULE) { - jl_serialize_value(s, t); - } } } } -static void jl_serialize_methtable_from_mod(ios_t *s, jl_module_t *m, jl_sym_t *name, jl_methtable_t *mt, int8_t iskw) +static void jl_serialize_methtable_from_mod(ios_t *s, jl_methtable_t *mt, int8_t iskw) { if (iskw) { if (!mt->kwsorter) return; assert(jl_is_gf(mt->kwsorter)); + assert(mt->module == jl_gf_mtable(mt->kwsorter)->module); mt = jl_gf_mtable(mt->kwsorter); assert(!mt->kwsorter); } - //XXX: we are reversing the list of methods due to #8652 - struct _chain { - jl_methlist_t *ml; - struct _chain *next; - } *chain = NULL; + assert(mt->module); jl_methlist_t *ml = mt->defs; while (ml != (void*)jl_nothing) { - if (jl_is_submodule(ml->func->linfo->module, jl_current_module)) { - struct _chain *link = (struct _chain*)alloca(sizeof(struct _chain)); - link->ml = ml; - link->next = chain; - chain = link; + if (module_in_worklist(ml->func->linfo->module)) { + jl_serialize_value(s, mt->module); + jl_serialize_value(s, mt->name); + write_int8(s, iskw); + jl_serialize_value(s, ml->sig); + jl_serialize_value(s, ml->func); + if (jl_is_svec(ml->tvars)) + jl_serialize_value(s, ml->tvars); + else + jl_serialize_value(s, jl_svec1(ml->tvars)); + write_int8(s, ml->isstaged); } ml = ml->next; } - while (chain) { - ml = chain->ml; - jl_serialize_value(s, m); - jl_serialize_value(s, name); - write_int8(s, iskw); - jl_serialize_value(s, ml->sig); - jl_serialize_value(s, ml->func); - if (jl_is_svec(ml->tvars)) - jl_serialize_value(s, ml->tvars); - else - jl_serialize_value(s, jl_svec1(ml->tvars)); - write_int8(s, ml->isstaged); - chain = chain->next; - } } static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) { - if (m == jl_current_module) return; + if (module_in_worklist(m)) return; size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - if (b->owner == m && b->value) { - if (jl_is_function(b->value)) { + if (b->owner == m && b->value && b->constp) { + if (jl_is_function(b->value) && jl_is_gf(b->value)) { jl_function_t *gf = (jl_function_t*)b->value; - if (jl_is_gf(gf)) { - jl_methtable_t *mt = jl_gf_mtable(gf); - jl_serialize_methtable_from_mod(s, m, b->name, mt, 0); - jl_serialize_methtable_from_mod(s, m, b->name, mt, 1); + jl_methtable_t *mt = jl_gf_mtable(gf); + if (mt->name == b->name && mt->module == m) { + jl_serialize_methtable_from_mod(s, mt, 0); + jl_serialize_methtable_from_mod(s, mt, 1); } - //TODO: look in datatype cache? } else if (jl_is_module(b->value)) { jl_module_t *child = (jl_module_t*)b->value; @@ -868,6 +921,7 @@ static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) } } +// serialize information about all of the modules accessible directly from Main void jl_serialize_mod_list(ios_t *s) { jl_module_t *m = jl_main_module; @@ -877,9 +931,9 @@ void jl_serialize_mod_list(ios_t *s) if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; if (b->owner == m && - b->value && - b->value != (jl_value_t*)jl_current_module && - jl_is_module(b->value)) { + b->value && b->constp && + jl_is_module(b->value) && + !module_in_worklist((jl_module_t*)b->value)) { jl_module_t *child = (jl_module_t*)b->value; if (child->name == b->name) { // this is the original/primary binding for the submodule @@ -932,6 +986,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt = jl_int64_type; else dt = jl_new_uninitialized_datatype(nf); + assert(tree_literal_values==NULL && mode != MODE_AST); + backref_list.items[pos] = dt; dt->size = size; dt->struct_decl = NULL; dt->instance = NULL; @@ -941,7 +997,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->pointerfree = (flags>>2)&1; if (!dt->abstract) { dt->ninitialized = read_uint16(s); - dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_LAMBDAS ? read_int32(s) : 0; + dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK ? read_int32(s) : 0; } else { dt->ninitialized = 0; @@ -949,18 +1005,16 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) } int has_instance = (flags>>3)&1; if (has_instance) { + assert(mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 dt->instance = jl_deserialize_value(s, &dt->instance); - jl_set_typeof(dt->instance, dt); + jl_gc_wb(dt, dt->instance); } - assert(tree_literal_values==NULL && mode != MODE_AST); - backref_list.items[pos] = dt; if (tag == 5) { + assert(pos > 0); + assert(mode != MODE_MODULE_POSTWORK); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)(uptrint_t)pos); - if (has_instance) { - arraylist_push(&flagref_list, &jl_astaggedvalue(dt->instance)->type); - arraylist_push(&flagref_list, (void*)(uptrint_t)-1); - } + dt->uid = jl_assign_type_uid(); // make sure this has a new uid early so it goes in the type cache correctly } if (nf > 0) { @@ -1002,6 +1056,7 @@ jl_array_t *jl_eqtable_put(jl_array_t *h, void *key, void *val); static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc); static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) { + assert(!ios_eof(s)); uint8_t tag = read_uint8(s); if (tag == Null_tag) return NULL; @@ -1019,7 +1074,7 @@ static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) isdatatype = !!(offs & 1); offs >>= 1; } - else if (mode == MODE_MODULE_LAMBDAS) { + else if (mode == MODE_MODULE_POSTWORK) { offs >>= 1; } assert(offs >= 0 && offs < backref_list.len); @@ -1102,13 +1157,14 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t int pos = backref_list.len; if (usetable) arraylist_push(&backref_list, NULL); - jl_value_t *aty = jl_deserialize_value(s, NULL); size_t *dims = (size_t*)alloca(ndims*sizeof(size_t)); for(i=0; i < ndims; i++) dims[i] = jl_unbox_long(jl_deserialize_value(s, NULL)); - jl_array_t *a = jl_new_array_for_deserialization((jl_value_t*)aty, ndims, dims, isunboxed, elsize); + jl_array_t *a = jl_new_array_for_deserialization((jl_value_t*)NULL, ndims, dims, isunboxed, elsize); if (usetable) backref_list.items[pos] = a; + jl_value_t *aty = jl_deserialize_value(s, &jl_astaggedvalue(a)->type); + jl_set_typeof(a, aty); if (!a->ptrarray) { size_t tot = jl_array_len(a) * a->elsize; ios_read(s, (char*)jl_array_data(a), tot); @@ -1120,10 +1176,6 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if (data[i]) jl_gc_wb(a, data[i]); } } - if (mode == MODE_MODULE) { - aty = jl_deserialize_value(s, &jl_astaggedvalue(a)->type); - assert(aty == jl_typeof(a)); - } return (jl_value_t*)a; } else if (vtag == (jl_value_t*)jl_expr_type || @@ -1161,6 +1213,18 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)tv; } else if (vtag == (jl_value_t*)jl_function_type) { + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) { + int ref_only = read_int8(s); + if (ref_only) { + int pos = backref_list.len; + arraylist_push(&backref_list, NULL); + jl_module_t *module = (jl_module_t*)jl_deserialize_value(s, NULL); + jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(s, NULL); + jl_value_t *f = jl_get_global(module, name); + backref_list.items[pos] = f; + return f; + } + } jl_function_t *f = (jl_function_t*)newobj((jl_value_t*)jl_function_type, NWORDS(sizeof(jl_function_t))); if (usetable) @@ -1209,6 +1273,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->inInference = 0; li->inCompile = 0; li->unspecialized = (jl_function_t*)jl_deserialize_value(s, (jl_value_t**)&li->unspecialized); + if (li->unspecialized) jl_gc_wb(li, li->unspecialized); li->functionID = 0; li->specFunctionID = 0; int32_t cfunc_llvm, func_llvm; @@ -1223,7 +1288,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t arraylist_push(&backref_list, NULL); jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL); int ref_only = 0; - if (mode == MODE_MODULE_LAMBDAS) { + if (mode == MODE_MODULE_POSTWORK) { ref_only = 1; } else if (mode == MODE_MODULE) { @@ -1290,18 +1355,32 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_value_t *sym = jl_deserialize_value(s, NULL); return jl_module_globalref(tree_enclosing_module, (jl_sym_t*)sym); } - else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)jl_globalref_type) { + else if (vtag == (jl_value_t*)jl_globalref_type) { + if (usetable) { + jl_value_t *v = jl_new_struct_uninit(jl_globalref_type); + arraylist_push(&backref_list, v); + jl_value_t* *data = jl_data_ptr(v); + data[0] = jl_deserialize_value(s, &data[0]); + data[1] = jl_deserialize_value(s, &data[1]); + return v; + } + else { + jl_value_t *mod = jl_deserialize_value(s, NULL); + jl_value_t *var = jl_deserialize_value(s, NULL); + return jl_module_globalref((jl_module_t*)mod, (jl_sym_t*)var); + } + } + else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { + int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); + jl_value_t *v = jl_gc_allocobj(sz); int pos = backref_list.len; if (usetable) - arraylist_push(&backref_list, NULL); - jl_datatype_t *dt; - if (vtag == (jl_value_t*)jl_globalref_type) - dt = jl_globalref_type; - else - dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); + arraylist_push(&backref_list, v); + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); + jl_set_typeof(v, dt); if (dt == jl_datatype_type) return jl_deserialize_datatype(s, pos, loc); - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && dt == jl_typename_type) { + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && dt == jl_typename_type) { int ref_only = read_uint8(s); if (ref_only) { jl_module_t *m = (jl_module_t*)jl_deserialize_value(s, NULL); @@ -1313,61 +1392,47 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t backref_list.items[pos] = v; return v; } + assert(mode != MODE_MODULE_POSTWORK); } size_t nf = jl_datatype_nfields(dt); - jl_value_t *v; if (nf == 0 && jl_datatype_size(dt)>0) { int nby = jl_datatype_size(dt); - char *data = (char*)alloca(nby); - ios_read(s, data, nby); - v = NULL; - if (dt == jl_int32_type) - v = jl_box_int32(*(int32_t*)data); - else if (dt == jl_int64_type) - v = jl_box_int64(*(int64_t*)data); - else if (dt == jl_bool_type) - v = jl_box_bool(*(int8_t*)data); - else { - switch (nby) { - case 1: v = jl_box8 (dt, *(int8_t *)data); break; - case 2: v = jl_box16(dt, *(int16_t*)data); break; - case 4: v = jl_box32(dt, *(int32_t*)data); break; - case 8: v = jl_box64(dt, *(int64_t*)data); break; - default: - v = (jl_value_t*)jl_gc_allocobj(nby); - jl_set_typeof(v, dt); - memcpy(jl_data_ptr(v), data, nby); - } - } - if (usetable) - backref_list.items[pos] = v; + ios_read(s, (char*)jl_data_ptr(v), nby); } else { - if (mode == MODE_AST && dt == jl_globalref_type) { - jl_value_t *mod = jl_deserialize_value(s, NULL); - jl_value_t *var = jl_deserialize_value(s, NULL); - return jl_module_globalref((jl_module_t*)mod, (jl_sym_t*)var); - } - v = jl_new_struct_uninit(dt); - if (usetable) - backref_list.items[pos] = v; char *data = (char*)jl_data_ptr(v); for(i=0; i < nf; i++) { - jl_set_nth_field(v, i, jl_deserialize_value(s, - (dt->fields[i].isptr) ? (jl_value_t**)(data+jl_field_offset(dt, i)) : NULL)); + if (dt->fields[i].size > 0) { + if (dt->fields[i].isptr) { + jl_value_t **fld = (jl_value_t**)(data+jl_field_offset(dt, i)); + *fld = jl_deserialize_value(s, fld); + } + else { + jl_set_nth_field(v, i, jl_deserialize_value(s, NULL)); + } + } + } + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { + if (jl_is_mtable(v)) + arraylist_push(&methtable_list, v); // will resort this table, later + if (dt == jl_typename_type) { + jl_typename_t* tn = (jl_typename_t*)v; + tn->uid = jl_assign_type_uid(); // make sure this has a new uid + tn->cache = jl_emptysvec; // the cache is refilled later (tag 5) + tn->linearcache = jl_emptysvec; // the cache is refilled later (tag 5) + } } - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && jl_is_mtable(v)) - arraylist_push(&methtable_list, v); - } - // TODO: put WeakRefs on the weak_refs list - if (mode == MODE_MODULE) { - dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); - assert((jl_value_t*)dt == jl_typeof(v)); } return v; } else if (vtag == (jl_value_t*)Singleton_tag) { - assert(mode != MODE_MODULE_LAMBDAS); + if (mode == MODE_MODULE_POSTWORK) { + uptrint_t pos = backref_list.len; + arraylist_push(&backref_list, NULL); + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); + backref_list.items[pos] = dt->instance; + return dt->instance; + } jl_value_t *v = (jl_value_t*)jl_gc_alloc_0w(); if (usetable) { uptrint_t pos = backref_list.len; @@ -1378,6 +1443,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t arraylist_push(&flagref_list, (void*)pos); } } + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); // no loc, since if dt is replaced, then dt->instance would be also + jl_set_typeof(v, dt); return v; } assert(0); @@ -1395,10 +1462,12 @@ void jl_deserialize_lambdas_from_mod(ios_t *s) int8_t iskw = read_int8(s); assert(jl_is_gf(gf)); if (iskw) { - if (!jl_gf_mtable(gf)->kwsorter) { - jl_gf_mtable(gf)->kwsorter = jl_new_generic_function(jl_gf_name(gf)); + jl_methtable_t* mt = jl_gf_mtable(gf); + if (!mt->kwsorter) { + mt->kwsorter = jl_new_generic_function(jl_gf_name(gf), mt->module); + jl_gc_wb(mt, mt->kwsorter); } - gf = jl_gf_mtable(gf)->kwsorter; + gf = mt->kwsorter; assert(jl_is_gf(gf)); } jl_tupletype_t *types = (jl_tupletype_t*)jl_deserialize_value(s, NULL); @@ -1423,30 +1492,133 @@ int jl_deserialize_verify_mod_list(ios_t *s) ios_read(s, name, len); name[len] = '\0'; uint64_t uuid = read_uint64(s); - jl_module_t *m = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol(name)); + jl_sym_t *sym = jl_symbol(name); + jl_module_t *m = (jl_module_t*)jl_get_global(jl_main_module, sym); if (!m) { - jl_printf(JL_STDERR, "error: Module %s must be loaded first\n", name); + static jl_value_t *require_func = NULL; + if (!require_func) + require_func = jl_get_global(jl_base_module, jl_symbol("require")); + JL_TRY { + jl_apply((jl_function_t*)require_func, (jl_value_t**)&sym, 1); + } + JL_CATCH { + ios_close(s); + jl_rethrow(); + } + m = (jl_module_t*)jl_get_global(jl_main_module, sym); + } + if (!m) { + jl_printf(JL_STDERR, "error: requiring \"%s\" did not define a corresponding module\n", name); return 0; } if (!jl_is_module(m)) { ios_close(s); - jl_errorf("typeassert: expected %s::Module", name); + jl_errorf("invalid module path (%s does not name a module)", name); } if (m->uuid != uuid) { - jl_printf(JL_STDERR, "error: Module %s uuid did not match cache file\n", name); + jl_printf(JL_STDERR, "warning: Module %s uuid did not match cache file\n", name); return 0; } } } -// --- entry points --- +jl_array_t *jl_module_init_order; + +static void jl_finalize_serializer(ios_t *f) { + size_t i, l; + // save module initialization order + if (jl_module_init_order != NULL) { + l = jl_array_len(jl_module_init_order); + for(i=0; i < l; i++) { + // verify that all these modules were saved + assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); + } + } + if (mode != MODE_MODULE) + jl_serialize_value(f, jl_module_init_order); -extern jl_array_t *jl_module_init_order; + // record list of reinitialization functions + l = reinit_list.len; + for (i = 0; i < l; i += 2) { + write_int32(f, (int)((uintptr_t) reinit_list.items[i])); + write_int32(f, (int)((uintptr_t) reinit_list.items[i+1])); + } + write_int32(f, -1); +} + +static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { + JL_TRY { + switch (how) { + case 1: { // rehash ObjectIdDict + jl_array_t **a = (jl_array_t**)&v->fieldptr[0]; + jl_idtable_rehash(a, jl_array_len(*a)); + jl_gc_wb(v, *a); + break; + } + case 2: { // reinsert module v into parent (const) + jl_module_t *mod = (jl_module_t*)v; + jl_binding_t *b = jl_get_binding_wr(mod->parent, mod->name); + jl_declare_constant(b); // this can throw + if (b->value != NULL) { + if (!jl_is_module(b->value)) { + jl_errorf("invalid redefinition of constant %s", mod->name->name); // this also throws + } + if (jl_generating_output() && jl_options.incremental) { + jl_errorf("error: cannot replace module %s during incremental compile", mod->name->name); + } + jl_printf(JL_STDERR, "Warning: replacing module %s\n", mod->name->name); + } + b->value = v; + jl_gc_wb_binding(b, v); + break; + } + default: + assert(0); + } + } + JL_CATCH { + jl_printf(JL_STDERR, "Warning: error reinitializing value "); + jl_static_show(JL_STDERR, v); + jl_printf(JL_STDERR, ":\n"); + jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_printf(JL_STDERR, "\n"); + } +} +static void jl_finalize_deserializer(ios_t *f) { + if (mode != MODE_MODULE) + jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + + // run reinitialization functions + int pos = read_int32(f); + while (pos != -1) { + jl_reinit_item(f, (jl_value_t*)backref_list.items[pos], read_int32(f)); + pos = read_int32(f); + } +} + +void jl_init_restored_modules() +{ + if (jl_module_init_order != NULL) { + jl_array_t *temp = jl_module_init_order; + jl_module_init_order = NULL; + JL_GC_PUSH1(&temp); + int i; + for(i=0; i < jl_array_len(temp); i++) { + jl_value_t *mod = jl_cellref(temp, i); + jl_module_run_initializer((jl_module_t*)mod); + } + JL_GC_POP(); + } +} + + +// --- entry points --- void jl_save_system_image_to_stream(ios_t *f) { - jl_gc_collect(1); - jl_gc_collect(0); + jl_gc_collect(1); // full + jl_gc_collect(0); // incremental (sweep finalizers) + JL_SIGATOMIC_BEGIN(); int en = jl_gc_enable(0); htable_reset(&backref_table, 250000); arraylist_new(&reinit_list, 0); @@ -1470,30 +1642,15 @@ void jl_save_system_image_to_stream(ios_t *f) jl_serialize_gv_syms(f, jl_get_root_symbol()); // serialize symbols with GlobalValue references jl_serialize_value(f, NULL); // signal the end of the symbols list - // save module initialization order - if (jl_module_init_order != NULL) { - size_t i; - for(i=0; i < jl_array_len(jl_module_init_order); i++) { - // verify that all these modules were saved - assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); - } - } - jl_serialize_value(f, jl_module_init_order); - write_int32(f, jl_get_t_uid_ctr()); write_int32(f, jl_get_gs_ctr()); - - // record reinitialization functions - for (i = 0; i < reinit_list.len; i += 2) { - write_int32(f, (int)((uintptr_t) reinit_list.items[i])); - write_int32(f, (int)((uintptr_t) reinit_list.items[i+1])); - } - write_int32(f, -1); + jl_finalize_serializer(f); // done with f htable_reset(&backref_table, 0); arraylist_free(&reinit_list); jl_gc_enable(en); + JL_SIGATOMIC_END(); } DLLEXPORT void jl_save_system_image(const char *fname) @@ -1502,8 +1659,10 @@ DLLEXPORT void jl_save_system_image(const char *fname) if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { jl_errorf("Cannot open system image file \"%s\" for writing.\n", fname); } + JL_SIGATOMIC_BEGIN(); jl_save_system_image_to_stream(&f); ios_close(&f); + JL_SIGATOMIC_END(); } DLLEXPORT ios_t *jl_create_system_image() @@ -1547,6 +1706,7 @@ DLLEXPORT void jl_preload_sysimg_so(const char *fname) void jl_restore_system_image_from_stream(ios_t *f) { + JL_SIGATOMIC_BEGIN(); int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_SYSTEM_IMAGE; @@ -1572,14 +1732,14 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_deserialize_globalvals(f); jl_deserialize_gv_syms(f); - jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + int uid_ctr = read_int32(f); + int gs_ctr = read_int32(f); + jl_finalize_deserializer(f); // done with f // cache builtin parametric types for(int i=0; i < jl_array_len(datatype_list); i++) { jl_value_t *v = jl_cellref(datatype_list, i); - uint32_t uid = ((jl_datatype_t*)v)->uid; jl_cache_type_((jl_datatype_t*)v); - ((jl_datatype_t*)v)->uid = uid; } datatype_list = NULL; @@ -1590,25 +1750,8 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_boot_file_loaded = 1; jl_init_box_caches(); - jl_set_t_uid_ctr(read_int32(f)); - jl_set_gs_ctr(read_int32(f)); - - // run reinitialization functions - int pos = read_int32(f); - while (pos != -1) { - jl_value_t *v = (jl_value_t*)backref_list.items[pos]; - switch (read_int32(f)) { - case 1: { - jl_array_t **a = (jl_array_t**)&v->fieldptr[0]; - jl_idtable_rehash(a, jl_array_len(*a)); - jl_gc_wb(v, *a); - break; - } - default: - assert(0); - } - pos = read_int32(f); - } + jl_set_t_uid_ctr(uid_ctr); + jl_set_gs_ctr(gs_ctr); //jl_printf(JL_STDERR, "backref_list.len = %d\n", backref_list.len); arraylist_free(&backref_list); @@ -1616,6 +1759,7 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_gc_enable(en); mode = last_mode; jl_update_all_fptrs(); + JL_SIGATOMIC_END(); } DLLEXPORT void jl_restore_system_image(const char *fname) @@ -1635,42 +1779,34 @@ DLLEXPORT void jl_restore_system_image(const char *fname) ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) jl_errorf("System image file \"%s\" not found\n", fname); + JL_SIGATOMIC_BEGIN(); jl_restore_system_image_from_stream(&f); ios_close(&f); + JL_SIGATOMIC_END(); } } DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) { ios_t f; + JL_SIGATOMIC_BEGIN(); ios_static_buffer(&f, (char*)buf, len); jl_restore_system_image_from_stream(&f); ios_close(&f); -} - -void jl_init_restored_modules() -{ - if (jl_module_init_order != NULL) { - jl_array_t *temp = jl_module_init_order; - jl_module_init_order = NULL; - JL_GC_PUSH1(&temp); - int i; - for(i=0; i < jl_array_len(temp); i++) { - jl_value_t *mod = jl_cellref(temp, i); - jl_module_run_initializer((jl_module_t*)mod); - } - JL_GC_POP(); - } + JL_SIGATOMIC_END(); } DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) { if (jl_is_expr(ast)) return jl_lam_body((jl_expr_t*)ast)->etype; + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; - if (li->module->constant_table == NULL) + if (li->module->constant_table == NULL) { li->module->constant_table = jl_alloc_cell_1d(0); + jl_gc_wb(li->module, li->module->constant_table); + } tree_literal_values = li->module->constant_table; ios_t src; jl_array_t *bytes = (jl_array_t*)ast; @@ -1682,11 +1818,13 @@ DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) jl_gc_enable(en); tree_literal_values = NULL; mode = last_mode; + JL_SIGATOMIC_END(); return rt; } DLLEXPORT jl_value_t *jl_compress_ast(jl_lambda_info_t *li, jl_value_t *ast) { + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; ios_t dest; @@ -1718,11 +1856,13 @@ DLLEXPORT jl_value_t *jl_compress_ast(jl_lambda_info_t *li, jl_value_t *ast) tree_enclosing_module = last_tem; jl_gc_enable(en); mode = last_mode; + JL_SIGATOMIC_END(); return v; } DLLEXPORT jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) { + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; jl_array_t *bytes = (jl_array_t*)data; @@ -1739,50 +1879,46 @@ DLLEXPORT jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) tree_literal_values = NULL; tree_enclosing_module = NULL; mode = last_mode; + JL_SIGATOMIC_END(); return v; } -DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) +DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { ios_t f; if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname); return 1; } - jl_module_t *lastmod = jl_current_module; - jl_current_module = mod; - jl_serialize_mod_list(&f); + serializer_worklist = worklist; + jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization) + + JL_SIGATOMIC_BEGIN(); + arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); - ptrhash_put(&backref_table, jl_main_module, (void*)(uintptr_t)0); + ptrhash_put(&backref_table, jl_main_module, HT_NOTFOUND + 1); backref_table_numel = 1; + jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_serialize_value(&f, mod->parent); - jl_serialize_value(&f, mod->name); - jl_serialize_value(&f, mod); + jl_serialize_value(&f, worklist); + jl_finalize_serializer(&f); // done with MODE_MODULE + reinit_list.len = 0; - mode = MODE_MODULE_LAMBDAS; + mode = MODE_MODULE_POSTWORK; jl_serialize_lambdas_from_mod(&f, jl_main_module); - jl_serialize_value(&f, NULL); + jl_serialize_value(&f, NULL); // signal end of lambdas + jl_finalize_serializer(&f); // done with f - // save module initialization order - if (jl_module_init_order != NULL) { - size_t i; - for(i=0; i < jl_array_len(jl_module_init_order); i++) { - // verify that all these modules were saved - assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); - } - } - jl_serialize_value(&f, jl_module_init_order); - - jl_current_module = lastmod; mode = last_mode; jl_gc_enable(en); htable_reset(&backref_table, 0); + arraylist_free(&reinit_list); ios_close(&f); + JL_SIGATOMIC_END(); return 0; } @@ -1790,21 +1926,17 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) jl_function_t *jl_method_cache_insert(jl_methtable_t *mt, jl_tupletype_t *type, jl_function_t *method); -DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) +static jl_array_t *_jl_restore_incremental(ios_t *f) { - ios_t f; - if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { - jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); + if (ios_eof(f)) { + ios_close(f); return NULL; } - if (ios_eof(&f)) { - ios_close(&f); - return NULL; - } - if (!jl_deserialize_verify_mod_list(&f)) { - ios_close(&f); + if (!jl_deserialize_verify_mod_list(f)) { + ios_close(f); return NULL; } + JL_SIGATOMIC_BEGIN(); arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); @@ -1813,19 +1945,15 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_module_t *parent = (jl_module_t*)jl_deserialize_value(&f, NULL); - jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(&f, NULL); - jl_binding_t *b = jl_get_binding_wr(parent, name); - jl_declare_constant(b); - if (b->value != NULL) { - jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); - } - b->value = jl_deserialize_value(&f, &b->value); + jl_array_t *last_module_init_order = jl_module_init_order; + jl_array_t *restored = NULL; + restored = (jl_array_t*)jl_deserialize_value(f, (jl_value_t**)&restored); size_t i = 0; while (i < flagref_list.len) { jl_value_t **loc = (jl_value_t**)flagref_list.items[i++]; - jl_value_t *v, *o = *loc; + int offs = (int)(intptr_t)flagref_list.items[i++]; + jl_value_t *v, *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; jl_datatype_t *dt; if (jl_is_datatype(o)) { dt = (jl_datatype_t*)o; @@ -1835,44 +1963,44 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) dt = (jl_datatype_t*)jl_typeof(o); v = o; } + assert(dt); jl_datatype_t *t = (jl_datatype_t*)jl_cache_type_(dt); - int offs = (int)(intptr_t)flagref_list.items[i++]; if (t != dt) { - jl_set_typeof(dt, (jl_value_t*)(ptrint_t)6); // invalidate the old value to help catch errors + jl_set_typeof(dt, (jl_value_t*)(ptrint_t)0x10); // invalidate the old value to help catch errors if ((jl_value_t*)dt == o) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } if (t->instance != v) { - jl_set_typeof(v, (jl_value_t*)(ptrint_t)4); // invalidate the old value to help catch errors + jl_set_typeof(v, (jl_value_t*)(ptrint_t)0x20); // invalidate the old value to help catch errors if (v == o) { - if (loc) *loc = v; - if (offs > 0) backref_list.items[offs] = v; + *loc = t->instance; + if (offs > 0) backref_list.items[offs] = t->instance; } } size_t j = i; while (j < flagref_list.len) { - if (flagref_list.items[j] == dt) { + jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; + int offs = (int)(intptr_t)flagref_list.items[j+1]; + jl_value_t *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; + if ((jl_value_t*)dt == o) { if (t != dt) { - jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; - int offs = (int)(intptr_t)flagref_list.items[j+1]; if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } - else if (flagref_list.items[j] == v) { + else if (v == o) { if (t->instance != v) { - jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; - int offs = (int)(intptr_t)flagref_list.items[j+1]; - if (loc) *loc = v; - if (offs > 0) backref_list.items[offs] = v; + *loc = t->instance; + if (offs > 0) backref_list.items[offs] = t->instance; } } else { j += 2; continue; } + // delete this item from the flagref list, so it won't be re-encountered later flagref_list.len -= 2; if (j >= flagref_list.len) break; @@ -1880,12 +2008,15 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) flagref_list.items[j+1] = flagref_list.items[flagref_list.len+1]; } } + jl_finalize_deserializer(f); // done with MODE_MODULE - mode = MODE_MODULE_LAMBDAS; - jl_deserialize_lambdas_from_mod(&f); - - jl_module_init_order = (jl_array_t*)jl_deserialize_value(&f, NULL); + // at this point, the AST is fully reconstructed, but still completely disconnected + // in postwork mode, all of the interconnects will be created + mode = MODE_MODULE_POSTWORK; + jl_deserialize_lambdas_from_mod(f); // hook up methods of external generic functions + jl_finalize_deserializer(f); // done with f + // Resort the internal method tables for (i = 0; i < methtable_list.len; i++) { jl_methtable_t *mt = (jl_methtable_t*)methtable_list.items[i]; jl_array_t *cache_targ = mt->cache_targ; @@ -1921,11 +2052,30 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) arraylist_free(&flagref_list); arraylist_free(&methtable_list); arraylist_free(&backref_list); - ios_close(&f); + ios_close(f); + JL_SIGATOMIC_END(); jl_init_restored_modules(); + jl_module_init_order = last_module_init_order; - return (jl_module_t*)b->value; + return restored; +} + +DLLEXPORT jl_array_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) +{ + ios_t f; + ios_static_buffer(&f, (char*)buf, sz); + return _jl_restore_incremental(&f); +} + +DLLEXPORT jl_array_t *jl_restore_incremental(const char *fname) +{ + ios_t f; + if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { + jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); + return NULL; + } + return _jl_restore_incremental(&f); } // --- init --- @@ -1941,10 +2091,11 @@ void jl_init_serializer(void) jl_function_type, jl_simplevector_type, jl_array_type, jl_expr_type, (void*)LongSymbol_tag, (void*)LongSvec_tag, (void*)LongExpr_tag, (void*)LiteralVal_tag, - (void*)SmallInt64_tag, (void*)UNUSED_tag, + (void*)SmallInt64_tag, (void*)SmallDataType_tag, (void*)Int32_tag, (void*)Array1d_tag, (void*)Singleton_tag, jl_module_type, jl_tvar_type, jl_lambda_info_type, (void*)CommonSym_tag, (void*)NearbyGlobal_tag, jl_globalref_type, + // everything above here represents a class of object rather only than a literal jl_emptysvec, jl_emptytuple, jl_false, jl_true, jl_nothing, jl_any_type, call_sym, goto_ifnot_sym, return_sym, body_sym, line_sym, @@ -2001,7 +2152,7 @@ void jl_init_serializer(void) jl_ANY_flag, jl_array_any_type, jl_intrinsic_type, jl_method_type, jl_methtable_type, jl_voidpointer_type, jl_newvarnode_type, jl_array_symbol_type, jl_anytuple_type, jl_tparam0(jl_anytuple_type), - + jl_typeof(jl_emptytuple), jl_symbol_type->name, jl_gensym_type->name, jl_tuple_typename, jl_ref_type->name, jl_pointer_type->name, jl_simplevector_type->name, jl_datatype_type->name, jl_uniontype_type->name, jl_array_type->name, diff --git a/src/gf.c b/src/gf.c index 96bebce10ad3e..b3b0ef408311b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -38,11 +38,12 @@ static jl_value_t *jl_apply_unspecialized(jl_function_t *meth, jl_value_t **args } -static jl_methtable_t *new_method_table(jl_sym_t *name) +static jl_methtable_t *new_method_table(jl_sym_t *name, jl_module_t *module) { jl_methtable_t *mt = (jl_methtable_t*)jl_gc_allocobj(sizeof(jl_methtable_t)); jl_set_typeof(mt, jl_methtable_type); mt->name = name; + mt->module = module; mt->defs = (jl_methlist_t*)jl_nothing; mt->cache = (jl_methlist_t*)jl_nothing; mt->cache_arg1 = (jl_array_t*)jl_nothing; @@ -1753,7 +1754,7 @@ jl_value_t *jl_gf_invoke(jl_function_t *gf, jl_tupletype_t *types, JL_GC_PUSH3(&tpenv, &newsig, &tt); tt = arg_type_tuple(args, nargs); if (m->invokes == (void*)jl_nothing) { - m->invokes = new_method_table(mt->name); + m->invokes = new_method_table(mt->name, mt->module); jl_gc_wb(m, m->invokes); update_max_args(m->invokes, tt); // this private method table has just this one definition @@ -1794,18 +1795,13 @@ void print_func_loc(JL_STREAM *s, jl_lambda_info_t *li) } } -void jl_initialize_generic_function(jl_function_t *f, jl_sym_t *name) -{ - f->fptr = jl_apply_generic; - f->env = (jl_value_t*)new_method_table(name); - jl_gc_wb(f, f->env); -} - -jl_function_t *jl_new_generic_function(jl_sym_t *name) +jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module) { jl_function_t *f = jl_new_closure(jl_apply_generic, NULL, NULL); JL_GC_PUSH1(&f); - jl_initialize_generic_function(f, name); + f->fptr = jl_apply_generic; + f->env = (jl_value_t*)new_method_table(name, module); + jl_gc_wb(f, f->env); JL_GC_POP(); return f; } diff --git a/src/init.c b/src/init.c index 1b6040e98aa1f..df2dd2670a725 100644 --- a/src/init.c +++ b/src/init.c @@ -169,7 +169,7 @@ static void jl_find_stack_bottom(void) // what to do on SIGINT DLLEXPORT void jl_sigint_action(void) { - if (exit_on_sigint) jl_exit(0); + if (exit_on_sigint) jl_exit(130); // 128+SIGINT jl_throw(jl_interrupt_exception); } @@ -363,7 +363,7 @@ static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guara } else { jl_signal_pending = 0; - if (exit_on_sigint) jl_exit(0); + if (exit_on_sigint) jl_exit(130); if ((DWORD)-1 == SuspendThread(hMainThread)) { //error jl_safe_printf("error: SuspendThread failed\n"); @@ -541,6 +541,7 @@ static void jl_uv_exitcleanup_walk(uv_handle_t *handle, void *arg) void jl_write_coverage_data(void); void jl_write_malloc_log(void); +static void julia_save(void); static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdown_queue_item *item) { @@ -549,8 +550,9 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow return rv; } -DLLEXPORT void jl_atexit_hook() +DLLEXPORT void jl_atexit_hook(int exitcode) { + if (exitcode == 0) julia_save(); #if defined(GC_FINAL_STATS) jl_print_gc_stats(JL_STDERR); #endif @@ -768,7 +770,7 @@ void *mach_segv_listener(void *arg) while (1) { int ret = mach_msg_server(exc_server,2048,segv_port,MACH_MSG_TIMEOUT_NONE); jl_safe_printf("mach_msg_server: %s\n", mach_error_string(ret)); - jl_exit(1); + jl_exit(128+SIGSEGV); } } @@ -1102,7 +1104,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_current_module = jl_core_module; jl_root_task->current_module = jl_current_module; - jl_load("boot.jl"); + jl_load("boot.jl", sizeof("boot.jl")); jl_get_builtin_hooks(); jl_boot_file_loaded = 1; jl_init_box_caches(); @@ -1282,33 +1284,48 @@ DLLEXPORT int jl_generating_output() void jl_compile_all(void); -DLLEXPORT void julia_save() +static void julia_save() { if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL) jl_compile_all(); - ios_t *s = NULL; - if (jl_options.outputo) - s = jl_create_system_image(); - - if (jl_options.outputji) { - if (s == NULL) { - jl_save_system_image(jl_options.outputji); - } - else { - ios_t f; - if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) - jl_errorf("Cannot open system image file \"%s\" for writing.\n", jl_options.outputji); - ios_write(&f, (const char*)s->buf, s->size); - ios_close(&f); + if (jl_options.incremental) { + jl_array_t *worklist = jl_module_init_order; + if (!worklist) { + jl_printf(JL_STDERR, "warning: incremental output requested, but no modules defined during run\n"); + return; } + if (jl_options.outputji) + jl_save_incremental(jl_options.outputji, worklist); + if (jl_options.outputbc) + jl_printf(JL_STDERR, "warning: incremental output to a .bc file is not implemented\n"); + if (jl_options.outputo) + jl_printf(JL_STDERR, "warning: incremental output to a .o file is not implemented\n"); } + else { + ios_t *s = NULL; + if (jl_options.outputo) + s = jl_create_system_image(); - if (jl_options.outputbc) - jl_dump_bitcode((char*)jl_options.outputbc); + if (jl_options.outputji) { + if (s == NULL) { + jl_save_system_image(jl_options.outputji); + } + else { + ios_t f; + if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) + jl_errorf("Cannot open system image file \"%s\" for writing.\n", jl_options.outputji); + ios_write(&f, (const char*)s->buf, s->size); + ios_close(&f); + } + } - if (jl_options.outputo) - jl_dump_objfile((char*)jl_options.outputo, 0, (const char*)s->buf, s->size); + if (jl_options.outputbc) + jl_dump_bitcode((char*)jl_options.outputbc); + + if (jl_options.outputo) + jl_dump_objfile((char*)jl_options.outputo, 0, (const char*)s->buf, s->size); + } } jl_function_t *jl_typeinf_func=NULL; diff --git a/src/jl_uv.c b/src/jl_uv.c index 0241169ea3741..e8f93685dcdb0 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -449,7 +449,7 @@ DLLEXPORT void jl_safe_printf(const char *fmt, ...) DLLEXPORT void jl_exit(int exitcode) { uv_tty_reset_mode(); - jl_atexit_hook(); + jl_atexit_hook(exitcode); exit(exitcode); } diff --git a/src/jltypes.c b/src/jltypes.c index 9e15ad69c6a7c..fc7bc198e30a9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3290,13 +3290,14 @@ void jl_init_types(void) jl_methtable_type = jl_new_datatype(jl_symbol("MethodTable"), jl_any_type, jl_emptysvec, - jl_svec(7, jl_symbol("name"), jl_symbol("defs"), + jl_svec(8, jl_symbol("name"), jl_symbol("defs"), jl_symbol("cache"), jl_symbol("cache_arg1"), - jl_symbol("cache_targ"), - jl_symbol("max_args"), jl_symbol("kwsorter")), - jl_svec(7, jl_sym_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type, jl_long_type, - jl_any_type), + jl_symbol("cache_targ"), jl_symbol("max_args"), + jl_symbol("kwsorter"), jl_symbol("module")), + jl_svec(8, jl_sym_type, jl_any_type, + jl_any_type, jl_any_type, + jl_any_type, jl_long_type, + jl_any_type, jl_any_type), 0, 1, 6); tv = jl_svec2(tvar("T"), tvar("N")); @@ -3381,6 +3382,7 @@ void jl_init_types(void) jl_svec(2, jl_module_type, jl_sym_type), 0, 0, 2); jl_svecset(jl_typename_type->types, 1, jl_module_type); + jl_svecset(jl_methtable_type->types, 7, jl_module_type); jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaStaticData"), diff --git a/src/julia.h b/src/julia.h index 7b0281f768170..da5afc4d3b7bf 100644 --- a/src/julia.h +++ b/src/julia.h @@ -351,6 +351,7 @@ typedef struct _jl_methtable_t { jl_array_t *cache_targ; ptrint_t max_args; // max # of non-vararg arguments in a signature jl_function_t *kwsorter; // keyword argument sorter function + jl_module_t *module; // used for incremental serialization to locate original binding #ifdef JL_GF_PROFILE int ncalls; #endif @@ -687,6 +688,7 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_cell_data(a) ((jl_value_t**)((jl_array_t*)a)->data) #define jl_string_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) +#define jl_string_len(s) (jl_array_len(((jl_array_t*)(s)->fieldptr[0]))) #define jl_iostr_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) #define jl_gf_mtable(f) ((jl_methtable_t*)((jl_function_t*)(f))->env) @@ -923,7 +925,7 @@ DLLEXPORT jl_sym_t *jl_gensym(void); DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, int32_t len); DLLEXPORT jl_sym_t *jl_get_root_symbol(void); jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); -jl_function_t *jl_new_generic_function(jl_sym_t *name); +jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module); void jl_add_method(jl_function_t *gf, jl_tupletype_t *types, jl_function_t *meth, jl_svec_t *tvars, int8_t isstaged); DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp, jl_value_t *bp_owner, @@ -1133,17 +1135,18 @@ DLLEXPORT void jl_init(const char *julia_home_dir); DLLEXPORT void jl_init_with_image(const char *julia_home_dir, const char *image_relative_path); DLLEXPORT int jl_is_initialized(void); DLLEXPORT int julia_trampoline(int argc, const char *argv[], int (*pmain)(int ac,char *av[])); -DLLEXPORT void jl_atexit_hook(void); +DLLEXPORT void jl_atexit_hook(int status); DLLEXPORT void NORETURN jl_exit(int status); DLLEXPORT void jl_preload_sysimg_so(const char *fname); -DLLEXPORT ios_t *jl_create_system_image(); +DLLEXPORT ios_t *jl_create_system_image(void); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); -DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod); -DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname); -void jl_init_restored_modules(); +DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t* worklist); +DLLEXPORT jl_array_t *jl_restore_incremental(const char *fname); +DLLEXPORT jl_array_t *jl_restore_incremental_from_buf(const char *buf, size_t sz); +void jl_init_restored_modules(void); // front end interface DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len); @@ -1193,7 +1196,7 @@ void jl_compile(jl_function_t *f); DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v); DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex); jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e); -DLLEXPORT jl_value_t *jl_load(const char *fname); +DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len); jl_value_t *jl_parse_eval_all(const char *fname, size_t len); jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam); jl_value_t *jl_interpret_toplevel_thunk_with(jl_lambda_info_t *lam, @@ -1275,7 +1278,7 @@ DLLEXPORT jl_value_t *jl_call2(jl_function_t *f, jl_value_t *a, jl_value_t *b); DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, jl_value_t *b, jl_value_t *c); // interfacing with Task runtime -DLLEXPORT void jl_yield(); +DLLEXPORT void jl_yield(void); // async signal handling ------------------------------------------------------ @@ -1525,11 +1528,12 @@ typedef struct { const char *outputbc; const char *outputo; const char *outputji; + int8_t incremental; } jl_options_t; extern DLLEXPORT jl_options_t jl_options; -DLLEXPORT int jl_generating_output(); +DLLEXPORT int jl_generating_output(void); // Settings for code_coverage and malloc_log // NOTE: if these numbers change, test/cmdlineargs.jl will have to be updated diff --git a/src/julia_internal.h b/src/julia_internal.h index 7175f37b44e21..018ea1308e77f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -100,13 +100,14 @@ jl_datatype_t *jl_inst_concrete_tupletype_v(jl_value_t **p, size_t np); jl_datatype_t *jl_inst_concrete_tupletype(jl_svec_t *p); void jl_set_datatype_super(jl_datatype_t *tt, jl_value_t *super); -void jl_initialize_generic_function(jl_function_t *f, jl_sym_t *name); void jl_add_constructors(jl_datatype_t *t); jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i); void jl_compute_field_offsets(jl_datatype_t *st); jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz); +extern jl_array_t *jl_module_init_order; + #ifdef JL_USE_INTEL_JITEVENTS extern char jl_using_intel_jitevents; #endif @@ -182,8 +183,6 @@ extern uv_lib_t *jl_crtdll_handle; extern uv_lib_t *jl_winsock_handle; #endif -DLLEXPORT void jl_atexit_hook(); - #if defined(_CPU_X86_) || defined(_CPU_X86_64_) #define HAVE_CPUID #endif diff --git a/src/toplevel.c b/src/toplevel.c index 9d0afe6d059da..f849e201bb631 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -48,6 +48,9 @@ void jl_add_standard_imports(jl_module_t *m) jl_module_t *jl_new_main_module(void) { + if (jl_generating_output() && jl_options.incremental) + jl_error("cannot call workspace() in incremental compile mode"); + // switch to a new top-level module if (jl_current_module != jl_main_module && jl_current_module != NULL) jl_error("Main can only be replaced from the top level"); @@ -70,8 +73,6 @@ jl_module_t *jl_new_main_module(void) return old_main; } -jl_array_t *jl_module_init_order = NULL; - // load time init procedure: in build mode, only record order void jl_module_load_time_initialize(jl_module_t *m) { @@ -111,9 +112,17 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) jl_module_t *parent_module = jl_current_module; jl_binding_t *b = jl_get_binding_wr(parent_module, name); jl_declare_constant(b); - if (b->value != NULL && !jl_generating_output()) { - // suppress warning "replacing module Core.Inference" during bootstrapping - jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + if (b->value != NULL) { + if (!jl_is_module(b->value)) { + jl_errorf("invalid redefinition of constant %s", name->name); + } + if (jl_generating_output() && jl_options.incremental) { + jl_errorf("error: cannot replace module %s during incremental compile", name->name); + } + if (!jl_generating_output()) { + // suppress warning "replacing module Core.Inference" during bootstrapping + jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + } } jl_module_t *newm = jl_new_module(name); newm->parent = parent_module; @@ -348,10 +357,7 @@ static jl_module_t *eval_import_path_(jl_array_t *args, int retrying) if (require_func == NULL && jl_base_module != NULL) require_func = jl_get_global(jl_base_module, jl_symbol("require")); if (require_func != NULL) { - jl_value_t *str = jl_cstr_to_string(var->name); - JL_GC_PUSH1(&str); - jl_apply((jl_function_t*)require_func, &str, 1); - JL_GC_POP(); + jl_apply((jl_function_t*)require_func, (jl_value_t**)&var, 1); return eval_import_path_(args, 1); } } @@ -570,7 +576,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) } JL_CATCH { jl_stop_parsing(); - fn = jl_pchar_to_string(fname, strlen(fname)); + fn = jl_pchar_to_string(fname, len); ln = jl_box_long(jl_lineno); jl_lineno = last_lineno; jl_filename = last_filename; @@ -589,7 +595,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) return result; } -jl_value_t *jl_load(const char *fname) +jl_value_t *jl_load(const char *fname, size_t len) { if (jl_current_module->istopmod) { jl_printf(JL_STDOUT, "%s\r\n", fname); @@ -605,7 +611,7 @@ jl_value_t *jl_load(const char *fname) if (jl_start_parsing_file(fpath) != 0) { jl_errorf("could not open file %s", fpath); } - jl_value_t *result = jl_parse_eval_all(fpath, strlen(fpath)); + jl_value_t *result = jl_parse_eval_all(fpath, len); if (fpath != fname) free(fpath); return result; } @@ -613,7 +619,7 @@ jl_value_t *jl_load(const char *fname) // load from filename given as a ByteString object DLLEXPORT jl_value_t *jl_load_(jl_value_t *str) { - return jl_load(jl_string_data(str)); + return jl_load(jl_string_data(str), jl_string_len(str)); } // type definition ------------------------------------------------------------ @@ -684,7 +690,8 @@ DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp, j if (bnd) bnd->constp = 1; if (*bp == NULL) { - gf = (jl_value_t*)jl_new_generic_function(name); + jl_module_t *module = (bnd ? bnd->owner : NULL); + gf = (jl_value_t*)jl_new_generic_function(name, module); *bp = gf; if (bp_owner) jl_gc_wb(bp_owner, gf); } @@ -714,6 +721,7 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t jl_svec_t *argdata, jl_function_t *f, jl_value_t *isstaged, jl_value_t *call_func, int iskw) { + jl_module_t *module = (bnd ? bnd->owner : NULL); // argdata is svec({types...}, svec(typevars...)) jl_tupletype_t *argtypes = (jl_tupletype_t*)jl_svecref(argdata,0); jl_svec_t *tvars = (jl_svec_t*)jl_svecref(argdata,1); @@ -785,8 +793,11 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t } } if (iskw) { - bp = (jl_value_t**)&((jl_methtable_t*)((jl_function_t*)gf)->env)->kwsorter; - bp_owner = (jl_value_t*)((jl_function_t*)gf)->env; + jl_methtable_t *mt = jl_gf_mtable(gf); + assert(!module); + module = mt->module; + bp = (jl_value_t**)&mt->kwsorter; + bp_owner = (jl_value_t*)mt; gf = *bp; } } @@ -819,7 +830,7 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t bnd->constp = 1; } if (*bp == NULL) { - gf = (jl_value_t*)jl_new_generic_function(name); + gf = (jl_value_t*)jl_new_generic_function(name, module); *bp = gf; if (bp_owner) jl_gc_wb(bp_owner, gf); } diff --git a/test/choosetests.jl b/test/choosetests.jl index 8dee768042cfa..a8a65d93b16ce 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -21,7 +21,7 @@ function choosetests(choices = []) "arrayops", "tuple", "subarray", "reduce", "reducedim", "random", "abstractarray", "intfuncs", "simdloop", "blas", "sparse", "bitarray", "copy", "math", "fastmath", "functional", - "operators", "path", "ccall", "parse", + "operators", "path", "ccall", "parse", "loading", "bigint", "sorting", "statistics", "spawn", "backtrace", "priorityqueue", "file", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", diff --git a/test/core.jl b/test/core.jl index 125743f0e592c..b7470c3318132 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1048,8 +1048,6 @@ let @test Sum < -0.69 end -include("test_sourcepath.jl") - # issue #2509 immutable Foo2509; foo::Int; end @test Foo2509(1) != Foo2509(2) diff --git a/test/loading.jl b/test/loading.jl new file mode 100644 index 0000000000000..cc6be3aa95f34 --- /dev/null +++ b/test/loading.jl @@ -0,0 +1,10 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +using Base.Test + +include("test_sourcepath.jl") +thefname = "the fname!//\\&\0\1*" +@test include_string("include_string_test() = @__FILE__", thefname)() == Base.source_path() +@test include_string("Base.source_path()", thefname) == Base.source_path() +@test basename(@__FILE__) == "loading.jl" +@test isabspath(@__FILE__) diff --git a/test/parallel.jl b/test/parallel.jl index 50dbd87bcae4c..27e70247ecf75 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # NOTE: worker processes cannot add more workers, only the client process can. -require("testdefs.jl") +using Base.Test if nworkers() < 3 remotecall_fetch(1, () -> addprocs(3 - nworkers())) diff --git a/test/runtests.jl b/test/runtests.jl index 6c40e8973454c..edd1a572296e7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,18 +2,16 @@ include("choosetests.jl") tests, net_on = choosetests(ARGS) -cd(dirname(@__FILE__)) do - n = 1 - if net_on - n = min(8, CPU_CORES, length(tests)) - n > 1 && addprocs(n; exeflags=`--check-bounds=yes --depwarn=error`) - blas_set_num_threads(1) - end +n = 1 +if net_on + n = min(8, CPU_CORES, length(tests)) + n > 1 && addprocs(n; exeflags=`--check-bounds=yes --depwarn=error`) + blas_set_num_threads(1) +end - @everywhere include("testdefs.jl") +@everywhere include("testdefs.jl") - reduce(propagate_errors, nothing, pmap(runtests, tests; err_retry=false, err_stop=true)) +reduce(propagate_errors, nothing, pmap(test->runtests(test), tests; err_retry=false, err_stop=true)) - @unix_only n > 1 && rmprocs(workers(), waitfor=5.0) - println(" \033[32;1mSUCCESS\033[0m") -end +@unix_only n > 1 && rmprocs(workers(), waitfor=5.0) +println(" \033[32;1mSUCCESS\033[0m") diff --git a/test/test_sourcepath.jl b/test/test_sourcepath.jl index 88af5cff0e680..837e89e126adf 100644 --- a/test/test_sourcepath.jl +++ b/test/test_sourcepath.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # source path in tasks -path = Base.source_path() +path = Base.source_path()::ByteString # this variable is leaked to the source script @test endswith(path, joinpath("test","test_sourcepath.jl")) @test yieldto(@task Base.source_path()) == path +@test isabspath(@__FILE__) diff --git a/test/testdefs.jl b/test/testdefs.jl index e7ddcda7635cf..15c97c57b438f 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -4,7 +4,7 @@ using Base.Test function runtests(name) @printf(" \033[1m*\033[0m \033[31m%-20s\033[0m", name) - tt = @elapsed Core.include(abspath("$name.jl")) + tt = @elapsed include("$name.jl") @printf(" in %6.2f seconds\n", tt) nothing end diff --git a/ui/repl.c b/ui/repl.c index 40dc6d584fbf5..2ebd44594690e 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -86,6 +86,7 @@ static const char opts[] = " --output-o name Generate an object file (including system image data)\n" " --output-ji name Generate a system image data file (.ji)\n" " --output-bc name Generate LLVM bitcode (.bc)\n\n" + " --output-incremental=no Generate an incremental output file (rather than complete)\n\n" // instrumentation options " --code-coverage={none|user|all}, --code-coverage\n" @@ -113,7 +114,8 @@ void parse_opts(int *argcp, char ***argvp) opt_handle_signals, opt_output_o, opt_output_ji, - opt_use_precompiled + opt_use_precompiled, + opt_incremental }; static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:Ob:"; static struct option longopts[] = { @@ -146,6 +148,7 @@ void parse_opts(int *argcp, char ***argvp) { "output-bc", required_argument, 0, opt_output_bc }, { "output-o", required_argument, 0, opt_output_o }, { "output-ji", required_argument, 0, opt_output_ji }, + { "output-incremental",required_argument, 0, opt_incremental }, { "depwarn", required_argument, 0, opt_depwarn }, { "inline", required_argument, 0, opt_inline }, { "math-mode", required_argument, 0, opt_math_mode }, @@ -330,6 +333,14 @@ void parse_opts(int *argcp, char ***argvp) jl_options.outputji = optarg; if (!imagepathspecified) jl_options.image_file = NULL; break; + case opt_incremental: + if (!strcmp(optarg,"yes")) + jl_options.incremental = 1; + else if (!strcmp(optarg,"no")) + jl_options.incremental = 0; + else + jl_errorf("julia: invalid argument to --output-incremental={yes|no} (%s)\n", optarg); + break; case opt_depwarn: if (!strcmp(optarg,"yes")) jl_options.depwarn = JL_OPTIONS_DEPWARN_ON; @@ -394,7 +405,7 @@ static int exec_program(char *program) jl_value_t *errs = jl_stderr_obj(); jl_value_t *e = jl_exception_in_transit; if (errs != NULL) { - jl_show(jl_stderr_obj(), e); + jl_show(errs, e); } else { jl_printf(JL_STDERR, "error during bootstrap:\n"); @@ -406,7 +417,7 @@ static int exec_program(char *program) JL_EH_POP(); return 1; } - jl_load(program); + jl_load(program, strlen(program)); } JL_CATCH { err = 1; @@ -513,8 +524,6 @@ static int true_main(int argc, char *argv[]) return 0; } -DLLEXPORT extern void julia_save(); - #ifndef _OS_WINDOWS_ int main(int argc, char *argv[]) { @@ -571,8 +580,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) } julia_init(imagepathspecified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); int ret = true_main(argc, (char**)argv); - julia_save(); - jl_atexit_hook(); + jl_atexit_hook(ret); return ret; }