Mercurial > octave
changeset 23126:2eb7d330eb7f
move JIT compiler files from corefcn to parse-tree directory
* jit-ir.cc, jit-ir.h, jit-typeinfo.cc, jit-typeinfo.h, jit-util.cc,
jit-util.h, pt-jit.cc, pt-jit.h: Move from libinterp/corefcn to
libinterp/parse-tree.
* libinterp/corefcn/module.mk, libinterp/parse-tree/module.mk: Update.
| author | John W. Eaton <jwe@octave.org> |
|---|---|
| date | Tue, 31 Jan 2017 17:43:58 -0500 |
| parents | ef0909c445df |
| children | 5a91168a30be |
| files | libinterp/corefcn/jit-ir.cc libinterp/corefcn/jit-ir.h libinterp/corefcn/jit-typeinfo.cc libinterp/corefcn/jit-typeinfo.h libinterp/corefcn/jit-util.cc libinterp/corefcn/jit-util.h libinterp/corefcn/module.mk libinterp/corefcn/pt-jit.cc libinterp/corefcn/pt-jit.h libinterp/parse-tree/jit-ir.cc libinterp/parse-tree/jit-ir.h libinterp/parse-tree/jit-typeinfo.cc libinterp/parse-tree/jit-typeinfo.h libinterp/parse-tree/jit-util.cc libinterp/parse-tree/jit-util.h libinterp/parse-tree/module.mk libinterp/parse-tree/pt-jit.cc libinterp/parse-tree/pt-jit.h |
| diffstat | 18 files changed, 8723 insertions(+), 8728 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/corefcn/jit-ir.cc Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,845 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -// defines required by llvm -#define __STDC_LIMIT_MACROS -#define __STDC_CONSTANT_MACROS - -#if defined (HAVE_CONFIG_H) -# include "config.h" -#endif - -#if defined (HAVE_LLVM) - -#include "jit-ir.h" - -#if defined (HAVE_LLVM_IR_FUNCTION_H) -# include <llvm/IR/BasicBlock.h> -# include <llvm/IR/Instructions.h> -#else -# include <llvm/BasicBlock.h> -# include <llvm/Instructions.h> -#endif - -#include "error.h" - -// -------------------- jit_factory -------------------- -jit_factory::~jit_factory (void) -{ - for (value_list::iterator iter = all_values.begin (); - iter != all_values.end (); ++iter) - delete *iter; -} - -void -jit_factory::track_value (jit_value *value) -{ - if (value->type ()) - mconstants.push_back (value); - all_values.push_back (value); -} - -// -------------------- jit_block_list -------------------- -void -jit_block_list::insert_after (iterator iter, jit_block *ablock) -{ - ++iter; - insert_before (iter, ablock); -} - -void -jit_block_list::insert_after (jit_block *loc, jit_block *ablock) -{ - insert_after (loc->location (), ablock); -} - -void -jit_block_list::insert_before (iterator iter, jit_block *ablock) -{ - iter = mlist.insert (iter, ablock); - ablock->stash_location (iter); -} - -void -jit_block_list::insert_before (jit_block *loc, jit_block *ablock) -{ - insert_before (loc->location (), ablock); -} - -void -jit_block_list::label (void) -{ - if (mlist.size ()) - { - jit_block *block = mlist.back (); - block->label (); - } -} - -std::ostream& -jit_block_list::print (std::ostream& os, const std::string& header) const -{ - os << "-------------------- " << header << " --------------------\n"; - return os << *this; -} - -std::ostream& -jit_block_list::print_dom (std::ostream& os) const -{ - os << "-------------------- dom info --------------------\n"; - for (const_iterator iter = begin (); iter != end (); ++iter) - { - assert (*iter); - (*iter)->print_dom (os); - } - os << std::endl; - - return os; -} - -void -jit_block_list::push_back (jit_block *b) -{ - mlist.push_back (b); - iterator iter = mlist.end (); - b->stash_location (--iter); -} - -std::ostream& -operator<<(std::ostream& os, const jit_block_list& blocks) -{ - for (jit_block_list::const_iterator iter = blocks.begin (); - iter != blocks.end (); ++iter) - { - assert (*iter); - (*iter)->print (os, 0); - } - return os << std::endl; -} - -// -------------------- jit_use -------------------- -jit_block * -jit_use::user_parent (void) const -{ - return muser->parent (); -} - -// -------------------- jit_value -------------------- -jit_value::~jit_value (void) -{ } - -jit_block * -jit_value::first_use_block (void) -{ - jit_use *use = first_use (); - while (use) - { - if (! isa<jit_error_check> (use->user ())) - return use->user_parent (); - - use = use->next (); - } - - return 0; -} - -void -jit_value::replace_with (jit_value *value) -{ - while (first_use ()) - { - jit_instruction *user = first_use ()->user (); - size_t idx = first_use ()->index (); - user->stash_argument (idx, value); - } -} - -#define JIT_METH(clname) \ - void \ - jit_ ## clname::accept (jit_ir_walker& walker) \ - { \ - walker.visit (*this); \ - } - -JIT_VISIT_IR_NOTEMPLATE -#undef JIT_METH - -std::ostream& -operator<< (std::ostream& os, const jit_value& value) -{ - return value.short_print (os); -} - -std::ostream& -jit_print (std::ostream& os, jit_value *avalue) -{ - if (avalue) - return avalue->print (os); - return os << "NULL"; -} - -// -------------------- jit_instruction -------------------- -void -jit_instruction::remove (void) -{ - if (mparent) - mparent->remove (mlocation); - resize_arguments (0); -} - -llvm::BasicBlock * -jit_instruction::parent_llvm (void) const -{ - return mparent->to_llvm (); -} - -std::ostream& -jit_instruction::short_print (std::ostream& os) const -{ - if (type ()) - jit_print (os, type ()) << ": "; - return os << "#" << mid; -} - -void -jit_instruction::do_construct_ssa (size_t start, size_t end) -{ - for (size_t i = start; i < end; ++i) - { - jit_value *arg = argument (i); - jit_variable *var = dynamic_cast<jit_variable *> (arg); - if (var && var->has_top ()) - stash_argument (i, var->top ()); - } -} - -// -------------------- jit_block -------------------- -void -jit_block::replace_with (jit_value *value) -{ - assert (isa<jit_block> (value)); - jit_block *block = static_cast<jit_block *> (value); - - jit_value::replace_with (block); - - while (ILIST_T::first_use ()) - { - jit_phi_incomming *incomming = ILIST_T::first_use (); - incomming->stash_value (block); - } -} - -void -jit_block::replace_in_phi (jit_block *ablock, jit_block *with) -{ - jit_phi_incomming *node = ILIST_T::first_use (); - while (node) - { - jit_phi_incomming *prev = node; - node = node->next (); - - if (prev->user_parent () == ablock) - prev->stash_value (with); - } -} - -jit_block * -jit_block::maybe_merge () -{ - if (successor_count () == 1 && successor (0) != this - && (successor (0)->use_count () == 1 || instructions.size () == 1)) - { - jit_block *to_merge = successor (0); - merge (*to_merge); - return to_merge; - } - - return 0; -} - -void -jit_block::merge (jit_block& block) -{ - // the merge block will contain a new terminator - jit_terminator *old_term = terminator (); - if (old_term) - old_term->remove (); - - bool was_empty = end () == begin (); - iterator merge_begin = end (); - if (! was_empty) - --merge_begin; - - instructions.splice (end (), block.instructions); - if (was_empty) - merge_begin = begin (); - else - ++merge_begin; - - // now merge_begin points to the start of the new instructions, we must - // update their parent information - for (iterator iter = merge_begin; iter != end (); ++iter) - { - jit_instruction *instr = *iter; - instr->stash_parent (this, iter); - } - - block.replace_with (this); -} - -jit_instruction * -jit_block::prepend (jit_instruction *instr) -{ - instructions.push_front (instr); - instr->stash_parent (this, instructions.begin ()); - return instr; -} - -jit_instruction * -jit_block::prepend_after_phi (jit_instruction *instr) -{ - // FIXME: Make this O(1) - for (iterator iter = begin (); iter != end (); ++iter) - { - jit_instruction *temp = *iter; - if (! isa<jit_phi> (temp)) - { - insert_before (iter, instr); - return instr; - } - } - - return append (instr); -} - -void -jit_block::internal_append (jit_instruction *instr) -{ - instructions.push_back (instr); - instr->stash_parent (this, --instructions.end ()); -} - -jit_instruction * -jit_block::insert_before (iterator loc, jit_instruction *instr) -{ - iterator iloc = instructions.insert (loc, instr); - instr->stash_parent (this, iloc); - return instr; -} - -jit_instruction * -jit_block::insert_after (iterator loc, jit_instruction *instr) -{ - ++loc; - iterator iloc = instructions.insert (loc, instr); - instr->stash_parent (this, iloc); - return instr; -} - -jit_terminator * -jit_block::terminator (void) const -{ - assert (this); - if (instructions.empty ()) - return 0; - - jit_instruction *last = instructions.back (); - return dynamic_cast<jit_terminator *> (last); -} - -bool -jit_block::branch_alive (jit_block *asucc) const -{ - return terminator ()->alive (asucc); -} - -jit_block * -jit_block::successor (size_t i) const -{ - jit_terminator *term = terminator (); - return term->successor (i); -} - -size_t -jit_block::successor_count (void) const -{ - jit_terminator *term = terminator (); - return term ? term->successor_count () : 0; -} - -llvm::BasicBlock * -jit_block::to_llvm (void) const -{ - return llvm::cast<llvm::BasicBlock> (llvm_value); -} - -std::ostream& -jit_block::print_dom (std::ostream& os) const -{ - short_print (os); - os << ":\n"; - os << " mid: " << mid << std::endl; - os << " predecessors: "; - for (jit_use *use = first_use (); use; use = use->next ()) - os << *use->user_parent () << " "; - os << std::endl; - - os << " successors: "; - for (size_t i = 0; i < successor_count (); ++i) - os << *successor (i) << " "; - os << std::endl; - - os << " idom: "; - if (idom) - os << *idom; - else - os << "NULL"; - os << std::endl; - os << " df: "; - for (df_iterator iter = df_begin (); iter != df_end (); ++iter) - os << **iter << " "; - os << std::endl; - - os << " dom_succ: "; - for (size_t i = 0; i < dom_succ.size (); ++i) - os << *dom_succ[i] << " "; - - return os << std::endl; -} - -void -jit_block::compute_df (size_t avisit_count) -{ - if (visited (avisit_count)) - return; - - if (use_count () >= 2) - { - for (jit_use *use = first_use (); use; use = use->next ()) - { - jit_block *runner = use->user_parent (); - while (runner != idom) - { - runner->mdf.insert (this); - runner = runner->idom; - } - } - } - - for (size_t i = 0; i < successor_count (); ++i) - successor (i)->compute_df (avisit_count); -} - -bool -jit_block::update_idom (size_t avisit_count) -{ - if (visited (avisit_count) || ! use_count ()) - return false; - - bool changed = false; - for (jit_use *use = first_use (); use; use = use->next ()) - { - jit_block *pred = use->user_parent (); - changed = pred->update_idom (avisit_count) || changed; - } - - jit_use *use = first_use (); - jit_block *new_idom = use->user_parent (); - use = use->next (); - - for (; use; use = use->next ()) - { - jit_block *pred = use->user_parent (); - jit_block *pidom = pred->idom; - if (pidom) - new_idom = idom_intersect (pidom, new_idom); - } - - if (idom != new_idom) - { - idom = new_idom; - return true; - } - - return changed; -} - -void -jit_block::label (size_t avisit_count, size_t& number) -{ - if (visited (avisit_count)) - return; - - for (jit_use *use = first_use (); use; use = use->next ()) - { - jit_block *pred = use->user_parent (); - pred->label (avisit_count, number); - } - - mid = number++; -} - -void -jit_block::pop_all (void) -{ - for (iterator iter = begin (); iter != end (); ++iter) - { - jit_instruction *instr = *iter; - instr->pop_variable (); - } -} - -std::ostream& -jit_block::print (std::ostream& os, size_t indent) const -{ - print_indent (os, indent); - short_print (os) << ": %pred = "; - for (jit_use *use = first_use (); use; use = use->next ()) - { - jit_block *pred = use->user_parent (); - os << *pred; - if (use->next ()) - os << ", "; - } - os << std::endl; - - for (const_iterator iter = begin (); iter != end (); ++iter) - { - jit_instruction *instr = *iter; - instr->print (os, indent + 1) << std::endl; - } - return os; -} - -jit_block * -jit_block::maybe_split (jit_factory& factory, jit_block_list& blocks, - jit_block *asuccessor) -{ - if (successor_count () > 1) - { - jit_terminator *term = terminator (); - size_t idx = term->successor_index (asuccessor); - jit_block *split = factory.create<jit_block> ("phi_split", mvisit_count); - - // place after this to ensure define before use in the blocks list - blocks.insert_after (this, split); - - term->stash_argument (idx, split); - jit_branch *br = split->append (factory.create<jit_branch> (asuccessor)); - replace_in_phi (asuccessor, split); - - if (alive ()) - { - split->mark_alive (); - br->infer (); - } - - return split; - } - - return this; -} - -void -jit_block::create_dom_tree (size_t avisit_count) -{ - if (visited (avisit_count)) - return; - - if (idom != this) - idom->dom_succ.push_back (this); - - for (size_t i = 0; i < successor_count (); ++i) - successor (i)->create_dom_tree (avisit_count); -} - -jit_block * -jit_block::idom_intersect (jit_block *i, jit_block *j) -{ - while (i && j && i != j) - { - while (i && i->id () > j->id ()) - i = i->idom; - - while (i && j && j->id () > i->id ()) - j = j->idom; - } - - return i ? i : j; -} - -// -------------------- jit_phi_incomming -------------------- - -jit_block * -jit_phi_incomming::user_parent (void) const -{ return muser->parent (); } - -// -------------------- jit_phi -------------------- -bool -jit_phi::prune (void) -{ - jit_block *p = parent (); - size_t new_idx = 0; - jit_value *unique = argument (1); - - for (size_t i = 0; i < argument_count (); ++i) - { - jit_block *inc = incomming (i); - if (inc->branch_alive (p)) - { - if (unique != argument (i)) - unique = 0; - - if (new_idx != i) - { - stash_argument (new_idx, argument (i)); - mincomming[new_idx].stash_value (inc); - } - - ++new_idx; - } - } - - if (new_idx != argument_count ()) - { - resize_arguments (new_idx); - mincomming.resize (new_idx); - } - - assert (argument_count () > 0); - if (unique) - { - replace_with (unique); - return true; - } - - return false; -} - -bool -jit_phi::infer (void) -{ - jit_block *p = parent (); - if (! p->alive ()) - return false; - - jit_type *infered = 0; - for (size_t i = 0; i < argument_count (); ++i) - { - jit_block *inc = incomming (i); - if (inc->branch_alive (p)) - infered = jit_typeinfo::join (infered, argument_type (i)); - } - - if (infered != type ()) - { - stash_type (infered); - return true; - } - - return false; -} - -llvm::PHINode * -jit_phi::to_llvm (void) const -{ - return llvm::cast<llvm::PHINode> (jit_value::to_llvm ()); -} - -// -------------------- jit_terminator -------------------- -size_t -jit_terminator::successor_index (const jit_block *asuccessor) const -{ - size_t scount = successor_count (); - for (size_t i = 0; i < scount; ++i) - if (successor (i) == asuccessor) - return i; - - panic_impossible (); -} - -bool -jit_terminator::infer (void) -{ - if (! parent ()->alive ()) - return false; - - bool changed = false; - for (size_t i = 0; i < malive.size (); ++i) - if (! malive[i] && check_alive (i)) - { - changed = true; - malive[i] = true; - successor (i)->mark_alive (); - } - - return changed; -} - -llvm::TerminatorInst * -jit_terminator::to_llvm (void) const -{ - return llvm::cast<llvm::TerminatorInst> (jit_value::to_llvm ()); -} - -// -------------------- jit_call -------------------- -bool -jit_call::needs_release (void) const -{ - if (type () && jit_typeinfo::get_release (type ()).valid ()) - { - for (jit_use *use = first_use (); use; use = use->next ()) - { - jit_assign *assign = dynamic_cast<jit_assign *> (use->user ()); - if (assign && assign->artificial ()) - return false; - } - - return true; - } - return false; -} - -bool -jit_call::infer (void) -{ - // FIXME: explain algorithm - for (size_t i = 0; i < argument_count (); ++i) - { - already_infered[i] = argument_type (i); - if (! already_infered[i]) - return false; - } - - jit_type *infered = moperation.result (already_infered); - if (! infered && use_count ()) - { - std::stringstream ss; - ss << "Missing overload in type inference for "; - print (ss, 0); - throw jit_fail_exception (ss.str ()); - } - - if (infered != type ()) - { - stash_type (infered); - return true; - } - - return false; -} - -// -------------------- jit_error_check -------------------- -std::string -jit_error_check::variable_to_string (variable v) -{ - switch (v) - { - case var_error_state: - return "error_state"; - case var_interrupt: - return "interrupt"; - default: - panic_impossible (); - } -} - -std::ostream& -jit_error_check::print (std::ostream& os, size_t indent) const -{ - print_indent (os, indent) << "error_check " << variable_to_string (mvariable) - << ", "; - - if (has_check_for ()) - os << "<for> " << *check_for () << ", "; - print_successor (os << "<normal> ", 1) << ", "; - return print_successor (os << "<error> ", 0); -} - -// -------------------- jit_magic_end -------------------- -jit_magic_end::context::context (jit_factory& factory, jit_value *avalue, - size_t aindex, size_t acount) - : value (avalue), index (factory.create<jit_const_index> (aindex)), - count (factory.create<jit_const_index> (acount)) -{ } - -jit_magic_end::jit_magic_end (const std::vector<context>& full_context) - : contexts (full_context) -{ - resize_arguments (contexts.size ()); - - size_t i; - std::vector<context>::const_iterator iter; - for (iter = contexts.begin (), i = 0; iter != contexts.end (); ++iter, ++i) - stash_argument (i, iter->value); -} - -jit_magic_end::context -jit_magic_end::resolve_context (void) const -{ - size_t idx; - for (idx = 0; idx < contexts.size (); ++idx) - { - jit_type *ctx_type = contexts[idx].value->type (); - if (! ctx_type || ctx_type->skip_paren ()) - break; - } - - if (idx >= contexts.size ()) - idx = 0; - - context ret = contexts[idx]; - ret.value = argument (idx); - return ret; -} - -bool -jit_magic_end::infer (void) -{ - jit_type *new_type = overload ().result (); - if (new_type != type ()) - { - stash_type (new_type); - return true; - } - - return false; -} - -std::ostream& -jit_magic_end::print (std::ostream& os, size_t indent) const -{ - context ctx = resolve_context (); - short_print (print_indent (os, indent)) << " (" << *ctx.value << ", "; - return os << *ctx.index << ", " << *ctx.count << ")"; -} - -const jit_function& -jit_magic_end::overload () const -{ - const context& ctx = resolve_context (); - return jit_typeinfo::end (ctx.value, ctx.index, ctx.count); -} - -#endif
--- a/libinterp/corefcn/jit-ir.h Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1444 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -#if ! defined (octave_jit_ir_h) -#define octave_jit_ir_h 1 - -#include "octave-config.h" - -#if defined (HAVE_LLVM) - -#include <list> -#include <stack> -#include <set> - -#include "jit-typeinfo.h" - -// The low level octave jit ir -// this ir is close to llvm, but contains information for doing type inference. -// We convert the octave parse tree to this IR directly. - -#define JIT_VISIT_IR_NOTEMPLATE \ - JIT_METH(block); \ - JIT_METH(branch); \ - JIT_METH(cond_branch); \ - JIT_METH(call); \ - JIT_METH(extract_argument); \ - JIT_METH(store_argument); \ - JIT_METH(return); \ - JIT_METH(phi); \ - JIT_METH(variable); \ - JIT_METH(error_check); \ - JIT_METH(assign) \ - JIT_METH(argument) \ - JIT_METH(magic_end) - -#define JIT_VISIT_IR_CONST \ - JIT_METH(const_bool); \ - JIT_METH(const_scalar); \ - JIT_METH(const_complex); \ - JIT_METH(const_index); \ - JIT_METH(const_string); \ - JIT_METH(const_range) - -#define JIT_VISIT_IR_CLASSES \ - JIT_VISIT_IR_NOTEMPLATE \ - JIT_VISIT_IR_CONST - -// forward declare all ir classes -#define JIT_METH(cname) \ - class jit_ ## cname; - -JIT_VISIT_IR_NOTEMPLATE - -#undef JIT_METH - -// ABCs which aren't included in JIT_VISIT_IR_ALL -class jit_instruction; -class jit_terminator; - -template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T = T, - bool QUOTE=false> -class jit_const; - -typedef jit_const<bool, jit_typeinfo::get_bool> jit_const_bool; -typedef jit_const<double, jit_typeinfo::get_scalar> jit_const_scalar; -typedef jit_const<Complex, jit_typeinfo::get_complex> jit_const_complex; -typedef jit_const<octave_idx_type, jit_typeinfo::get_index> jit_const_index; - -typedef jit_const<std::string, jit_typeinfo::get_string, const std::string&, - true> jit_const_string; -typedef jit_const<jit_range, jit_typeinfo::get_range, const jit_range&> -jit_const_range; - -class jit_ir_walker; -class jit_use; - -// Creates and tracks memory for jit_value and subclasses. -// Memory managment is simple, all values that are created live as long as the -// factory. -class -jit_factory -{ - typedef std::list<jit_value *> value_list; - -public: - - ~jit_factory (void); - - const value_list& constants (void) const { return mconstants; } - - template <typename T> - T *create (void) - { - T *ret = new T (); - track_value (ret); - return ret; - } - -#define DECL_ARG(n) const ARG ## n& arg ## n - -#define JIT_CREATE(N) \ - template <typename T, OCT_MAKE_DECL_LIST (typename, ARG, N)> \ - T *create (OCT_MAKE_LIST (DECL_ARG, N)) \ - { \ - T *ret = new T (OCT_MAKE_ARG_LIST (arg, N)); \ - track_value (ret); \ - return ret; \ - } - - JIT_CREATE (1) - JIT_CREATE (2) - JIT_CREATE (3) - JIT_CREATE (4) - -#undef JIT_CREATE -#undef DECL_ARG - -private: - - void track_value (jit_value *v); - - value_list all_values; - - value_list mconstants; -}; - -// A list of basic blocks (jit_block) which form some body of code. -// -// We do not directly inherit from std::list because we need to update the -// blocks stashed location in push_back and insert. -class -jit_block_list -{ -public: - typedef std::list<jit_block *>::iterator iterator; - typedef std::list<jit_block *>::const_iterator const_iterator; - - jit_block *back (void) const { return mlist.back (); } - - iterator begin (void) { return mlist.begin (); } - - const_iterator begin (void) const { return mlist.begin (); } - - iterator end (void) { return mlist.end (); } - - const_iterator end (void) const { return mlist.end (); } - - iterator erase (iterator iter) { return mlist.erase (iter); } - - jit_block *front (void) const { return mlist.front (); } - - void insert_after (iterator iter, jit_block *ablock); - - void insert_after (jit_block *loc, jit_block *ablock); - - void insert_before (iterator iter, jit_block *ablock); - - void insert_before (jit_block *loc, jit_block *ablock); - - void label (void); - - std::ostream& print (std::ostream& os, const std::string& header) const; - - std::ostream& print_dom (std::ostream& os) const; - - void push_back (jit_block *b); -private: - std::list<jit_block *> mlist; -}; - -std::ostream& operator<<(std::ostream& os, const jit_block_list& blocks); - -class -jit_value : public jit_internal_list<jit_value, jit_use> -{ -public: - jit_value (void) : llvm_value (0), ty (0), mlast_use (0), - min_worklist (false) { } - - virtual ~jit_value (void); - - bool in_worklist (void) const - { - return min_worklist; - } - - void stash_in_worklist (bool ain_worklist) - { - min_worklist = ain_worklist; - } - - // The block of the first use which is not a jit_error_check - // So this is not necessarily first_use ()->parent (). - jit_block *first_use_block (void); - - // replace all uses with - virtual void replace_with (jit_value *value); - - jit_type *type (void) const { return ty; } - - llvm::Type *type_llvm (void) const - { - return ty ? ty->to_llvm () : 0; - } - - const std::string& type_name (void) const - { - return ty->name (); - } - - void stash_type (jit_type *new_ty) { ty = new_ty; } - - std::string print_string (void) - { - std::stringstream ss; - print (ss); - return ss.str (); - } - - jit_instruction *last_use (void) const { return mlast_use; } - - void stash_last_use (jit_instruction *alast_use) - { - mlast_use = alast_use; - } - - virtual bool needs_release (void) const { return false; } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const = 0; - - virtual std::ostream& short_print (std::ostream& os) const - { return print (os); } - - virtual void accept (jit_ir_walker& walker) = 0; - - bool has_llvm (void) const - { - return llvm_value; - } - - llvm::Value *to_llvm (void) const - { - assert (llvm_value); - return llvm_value; - } - - void stash_llvm (llvm::Value *compiled) - { - llvm_value = compiled; - } - -protected: - std::ostream& print_indent (std::ostream& os, size_t indent = 0) const - { - for (size_t i = 0; i < indent * 8; ++i) - os << " "; - return os; - } - - llvm::Value *llvm_value; -private: - jit_type *ty; - jit_instruction *mlast_use; - bool min_worklist; -}; - -std::ostream& operator<< (std::ostream& os, const jit_value& value); -std::ostream& jit_print (std::ostream& os, jit_value *avalue); - -class -jit_use : public jit_internal_node<jit_value, jit_use> -{ -public: - // some compilers don't allow us to use jit_internal_node without template - // paremeters - typedef jit_internal_node<jit_value, jit_use> PARENT_T; - - jit_use (void) : muser (0), mindex (0) { } - - // we should really have a move operator, but not until c++11 :( - jit_use (const jit_use& use) : muser (0), mindex (0) - { - *this = use; - } - - jit_use& operator= (const jit_use& use) - { - stash_value (use.value (), use.user (), use.index ()); - return *this; - } - - size_t index (void) const { return mindex; } - - jit_instruction *user (void) const { return muser; } - - jit_block *user_parent (void) const; - - std::list<jit_block *> user_parent_location (void) const; - - void stash_value (jit_value *avalue, jit_instruction *auser = 0, - size_t aindex = -1) - { - PARENT_T::stash_value (avalue); - mindex = aindex; - muser = auser; - } -private: - jit_instruction *muser; - size_t mindex; -}; - -class -jit_instruction : public jit_value -{ -public: - // FIXME: this code could be so much pretier with varadic templates... - jit_instruction (void) : mid (next_id ()), mparent (0) - { } - - jit_instruction (size_t nargs) : mid (next_id ()), mparent (0) - { - already_infered.reserve (nargs); - marguments.reserve (nargs); - } - -#define STASH_ARG(i) stash_argument (i, arg ## i); - -#define JIT_INSTRUCTION_CTOR(N) \ - jit_instruction (OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ - : already_infered (N), marguments (N), mid (next_id ()), mparent (0) \ - { \ - OCT_ITERATE_MACRO (STASH_ARG, N); \ - } - - JIT_INSTRUCTION_CTOR(1) - JIT_INSTRUCTION_CTOR(2) - JIT_INSTRUCTION_CTOR(3) - JIT_INSTRUCTION_CTOR(4) - -#undef STASH_ARG -#undef JIT_INSTRUCTION_CTOR - - jit_instruction (const std::vector<jit_value *>& aarguments) - : already_infered (aarguments.size ()), marguments (aarguments.size ()), - mid (next_id ()), mparent (0) - { - for (size_t i = 0; i < aarguments.size (); ++i) - stash_argument (i, aarguments[i]); - } - - static void reset_ids (void) - { - next_id (true); - } - - jit_value *argument (size_t i) const - { - return marguments[i].value (); - } - - llvm::Value *argument_llvm (size_t i) const - { - assert (argument (i)); - return argument (i)->to_llvm (); - } - - jit_type *argument_type (size_t i) const - { - return argument (i)->type (); - } - - llvm::Type *argument_type_llvm (size_t i) const - { - assert (argument (i)); - return argument_type (i)->to_llvm (); - } - - std::ostream& print_argument (std::ostream& os, size_t i) const - { - if (argument (i)) - return argument (i)->short_print (os); - else - return os << "NULL"; - } - - void stash_argument (size_t i, jit_value *arg) - { - marguments[i].stash_value (arg, this, i); - } - - void push_argument (jit_value *arg) - { - marguments.push_back (jit_use ()); - stash_argument (marguments.size () - 1, arg); - already_infered.push_back (0); - } - - size_t argument_count (void) const - { - return marguments.size (); - } - - void resize_arguments (size_t acount, jit_value *adefault = 0) - { - size_t old = marguments.size (); - marguments.resize (acount); - already_infered.resize (acount); - - if (adefault) - for (size_t i = old; i < acount; ++i) - stash_argument (i, adefault); - } - - const std::vector<jit_use>& arguments (void) const { return marguments; } - - // argument types which have been infered already - const std::vector<jit_type *>& argument_types (void) const - { return already_infered; } - - virtual void push_variable (void) { } - - virtual void pop_variable (void) { } - - virtual void construct_ssa (void) - { - do_construct_ssa (0, argument_count ()); - } - - virtual bool infer (void) { return false; } - - void remove (void); - - virtual std::ostream& short_print (std::ostream& os) const; - - jit_block *parent (void) const { return mparent; } - - std::list<jit_instruction *>::iterator location (void) const - { - return mlocation; - } - - llvm::BasicBlock *parent_llvm (void) const; - - void stash_parent (jit_block *aparent, - std::list<jit_instruction *>::iterator alocation) - { - mparent = aparent; - mlocation = alocation; - } - - size_t id (void) const { return mid; } -protected: - - // Do SSA replacement on arguments in [start, end) - void do_construct_ssa (size_t start, size_t end); - - std::vector<jit_type *> already_infered; -private: - static size_t next_id (bool reset = false) - { - static size_t ret = 0; - if (reset) - return ret = 0; - - return ret++; - } - - std::vector<jit_use> marguments; - - size_t mid; - jit_block *mparent; - std::list<jit_instruction *>::iterator mlocation; -}; - -// defnie accept methods for subclasses -#define JIT_VALUE_ACCEPT \ - virtual void accept (jit_ir_walker& walker); - -// for use as a dummy argument during conversion to LLVM -class -jit_argument : public jit_value -{ -public: - jit_argument (jit_type *atype, llvm::Value *avalue) - { - stash_type (atype); - stash_llvm (avalue); - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent); - return jit_print (os, type ()) << ": DUMMY"; - } - - JIT_VALUE_ACCEPT; -}; - -template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T, bool QUOTE> -class -jit_const : public jit_value -{ -public: - typedef PASS_T pass_t; - - jit_const (PASS_T avalue) : mvalue (avalue) - { - stash_type (EXTRACT_T ()); - } - - PASS_T value (void) const { return mvalue; } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent); - jit_print (os, type ()) << ": "; - if (QUOTE) - os << "\""; - os << mvalue; - if (QUOTE) - os << "\""; - return os; - } - - JIT_VALUE_ACCEPT; -private: - T mvalue; -}; - -class jit_phi_incomming; - -class -jit_block : public jit_value, public jit_internal_list<jit_block, - jit_phi_incomming> -{ - typedef jit_internal_list<jit_block, jit_phi_incomming> ILIST_T; -public: - typedef std::list<jit_instruction *> instruction_list; - typedef instruction_list::iterator iterator; - typedef instruction_list::const_iterator const_iterator; - - typedef std::set<jit_block *> df_set; - typedef df_set::const_iterator df_iterator; - - static const size_t NO_ID = static_cast<size_t> (-1); - - jit_block (const std::string& aname, size_t avisit_count = 0) - : mvisit_count (avisit_count), mid (NO_ID), idom (0), mname (aname), - malive (false) - { } - - virtual void replace_with (jit_value *value); - - void replace_in_phi (jit_block *ablock, jit_block *with); - - // we have a new internal list, but we want to stay compatible with jit_value - jit_use *first_use (void) const { return jit_value::first_use (); } - - size_t use_count (void) const { return jit_value::use_count (); } - - // if a block is alive, then it might be visited during execution - bool alive (void) const { return malive; } - - void mark_alive (void) { malive = true; } - - // If we can merge with a successor, do so and return the now empty block - jit_block *maybe_merge (); - - // merge another block into this block, leaving the merge block empty - void merge (jit_block& merge); - - const std::string& name (void) const { return mname; } - - jit_instruction *prepend (jit_instruction *instr); - - jit_instruction *prepend_after_phi (jit_instruction *instr); - - template <typename T> - T *append (T *instr) - { - internal_append (instr); - return instr; - } - - jit_instruction *insert_before (iterator loc, jit_instruction *instr); - - jit_instruction *insert_before (jit_instruction *loc, jit_instruction *instr) - { - return insert_before (loc->location (), instr); - } - - jit_instruction *insert_after (iterator loc, jit_instruction *instr); - - jit_instruction *insert_after (jit_instruction *loc, jit_instruction *instr) - { - return insert_after (loc->location (), instr); - } - - iterator remove (iterator iter) - { - jit_instruction *instr = *iter; - iter = instructions.erase (iter); - instr->stash_parent (0, instructions.end ()); - return iter; - } - - jit_terminator *terminator (void) const; - - // is the jump from pred alive? - bool branch_alive (jit_block *asucc) const; - - jit_block *successor (size_t i) const; - - size_t successor_count (void) const; - - iterator begin (void) { return instructions.begin (); } - - const_iterator begin (void) const { return instructions.begin (); } - - iterator end (void) { return instructions.end (); } - - const_iterator end (void) const { return instructions.end (); } - - iterator phi_begin (void); - - iterator phi_end (void); - - iterator nonphi_begin (void); - - // must label before id is valid - size_t id (void) const { return mid; } - - // dominance frontier - const df_set& df (void) const { return mdf; } - - df_iterator df_begin (void) const { return mdf.begin (); } - - df_iterator df_end (void) const { return mdf.end (); } - - // label with a RPO walk - void label (void) - { - size_t number = 0; - label (mvisit_count, number); - } - - void label (size_t avisit_count, size_t& number); - - // See for idom computation algorithm - // Cooper, Keith D.; Harvey, Timothy J; and Kennedy, Ken (2001). - // "A Simple, Fast Dominance Algorithm" - void compute_idom (jit_block& entry_block) - { - bool changed; - entry_block.idom = &entry_block; - do - changed = update_idom (mvisit_count); - while (changed); - } - - // compute dominance frontier - void compute_df (void) - { - compute_df (mvisit_count); - } - - void create_dom_tree (void) - { - create_dom_tree (mvisit_count); - } - - jit_block *dom_successor (size_t idx) const - { - return dom_succ[idx]; - } - - size_t dom_successor_count (void) const - { - return dom_succ.size (); - } - - // call pop_varaible on all instructions - void pop_all (void); - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; - - jit_block *maybe_split (jit_factory& factory, jit_block_list& blocks, - jit_block *asuccessor); - - jit_block *maybe_split (jit_factory& factory, jit_block_list& blocks, - jit_block& asuccessor) - { - return maybe_split (factory, blocks, &asuccessor); - } - - // print dominator infomration - std::ostream& print_dom (std::ostream& os) const; - - virtual std::ostream& short_print (std::ostream& os) const - { - os << mname; - if (mid != NO_ID) - os << mid; - else - os << "!"; - return os; - } - - llvm::BasicBlock *to_llvm (void) const; - - std::list<jit_block *>::iterator location (void) const - { return mlocation; } - - void stash_location (std::list<jit_block *>::iterator alocation) - { mlocation = alocation; } - - // used to prevent visiting the same node twice in the graph - size_t visit_count (void) const { return mvisit_count; } - - // check if this node has been visited yet at the given visit count. - // If we have not been visited yet, mark us as visited. - bool visited (size_t avisit_count) - { - if (mvisit_count <= avisit_count) - { - mvisit_count = avisit_count + 1; - return false; - } - - return true; - } - - jit_instruction *front (void) { return instructions.front (); } - - jit_instruction *back (void) { return instructions.back (); } - - JIT_VALUE_ACCEPT; -private: - void internal_append (jit_instruction *instr); - - void compute_df (size_t avisit_count); - - bool update_idom (size_t avisit_count); - - void create_dom_tree (size_t avisit_count); - - static jit_block *idom_intersect (jit_block *i, jit_block *j); - - size_t mvisit_count; - size_t mid; - jit_block *idom; - df_set mdf; - std::vector<jit_block *> dom_succ; - std::string mname; - instruction_list instructions; - bool malive; - std::list<jit_block *>::iterator mlocation; -}; - -// keeps track of phi functions that use a block on incomming edges -class -jit_phi_incomming : public jit_internal_node<jit_block, jit_phi_incomming> -{ -public: - jit_phi_incomming (void) : muser (0) { } - - jit_phi_incomming (jit_phi *auser) : muser (auser) { } - - jit_phi_incomming (const jit_phi_incomming& use) - { - *this = use; - } - - jit_phi_incomming& operator= (const jit_phi_incomming& use) - { - stash_value (use.value ()); - muser = use.muser; - return *this; - } - - jit_phi *user (void) const { return muser; } - - jit_block *user_parent (void) const; -private: - jit_phi *muser; -}; - -// A non-ssa variable -class -jit_variable : public jit_value -{ -public: - jit_variable (const std::string& aname) : mname (aname), mlast_use (0) { } - - const std::string &name (void) const { return mname; } - - // manipulate the value_stack, for use during SSA construction. The top of - // the value stack represents the current value for this variable - bool has_top (void) const - { - return ! value_stack.empty (); - } - - jit_value *top (void) const - { - return value_stack.top (); - } - - void push (jit_instruction *v) - { - value_stack.push (v); - mlast_use = v; - } - - void pop (void) - { - value_stack.pop (); - } - - jit_instruction *last_use (void) const - { - return mlast_use; - } - - void stash_last_use (jit_instruction *instr) - { - mlast_use = instr; - } - - // blocks in which we are used - void use_blocks (jit_block::df_set& result) - { - jit_use *use = first_use (); - while (use) - { - result.insert (use->user_parent ()); - use = use->next (); - } - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - return print_indent (os, indent) << mname; - } - - JIT_VALUE_ACCEPT; -private: - std::string mname; - std::stack<jit_value *> value_stack; - jit_instruction *mlast_use; -}; - -class -jit_assign_base : public jit_instruction -{ -public: - jit_assign_base (jit_variable *adest) : jit_instruction (), mdest (adest) { } - - jit_assign_base (jit_variable *adest, size_t npred) : jit_instruction (npred), - mdest (adest) { } - - jit_assign_base (jit_variable *adest, jit_value *arg0, jit_value *arg1) - : jit_instruction (arg0, arg1), mdest (adest) { } - - jit_variable *dest (void) const { return mdest; } - - virtual void push_variable (void) - { - mdest->push (this); - } - - virtual void pop_variable (void) - { - mdest->pop (); - } - - virtual std::ostream& short_print (std::ostream& os) const - { - if (type ()) - jit_print (os, type ()) << ": "; - - dest ()->short_print (os); - return os << "#" << id (); - } -private: - jit_variable *mdest; -}; - -class -jit_assign : public jit_assign_base -{ -public: - jit_assign (jit_variable *adest, jit_value *asrc) - : jit_assign_base (adest, adest, asrc), martificial (false) { } - - jit_value *overwrite (void) const - { - return argument (0); - } - - jit_value *src (void) const - { - return argument (1); - } - - // variables don't get modified in an SSA, but COW requires we modify - // variables. An artificial assign is for when a variable gets modified. We - // need an assign in the SSA, but the reference counts shouldn't be updated. - bool artificial (void) const { return martificial; } - - void mark_artificial (void) { martificial = true; } - - virtual bool infer (void) - { - jit_type *stype = src ()->type (); - if (stype != type()) - { - stash_type (stype); - return true; - } - - return false; - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent) << *this << " = " << *src (); - - if (artificial ()) - os << " [artificial]"; - - return os; - } - - JIT_VALUE_ACCEPT; -private: - bool martificial; -}; - -class -jit_phi : public jit_assign_base -{ -public: - jit_phi (jit_variable *adest, size_t npred) - : jit_assign_base (adest, npred) - { - mincomming.reserve (npred); - } - - // removes arguments form dead incomming jumps - bool prune (void); - - void add_incomming (jit_block *from, jit_value *value) - { - push_argument (value); - mincomming.push_back (jit_phi_incomming (this)); - mincomming[mincomming.size () - 1].stash_value (from); - } - - jit_block *incomming (size_t i) const - { - return mincomming[i].value (); - } - - llvm::BasicBlock *incomming_llvm (size_t i) const - { - return incomming (i)->to_llvm (); - } - - virtual void construct_ssa (void) { } - - virtual bool infer (void); - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - std::stringstream ss; - print_indent (ss, indent); - short_print (ss) << " phi "; - std::string ss_str = ss.str (); - std::string indent_str (ss_str.size (), ' '); - os << ss_str; - - for (size_t i = 0; i < argument_count (); ++i) - { - if (i > 0) - os << indent_str; - os << "| "; - - os << *incomming (i) << " -> "; - os << *argument (i); - - if (i + 1 < argument_count ()) - os << std::endl; - } - - return os; - } - - llvm::PHINode *to_llvm (void) const; - - JIT_VALUE_ACCEPT; -private: - std::vector<jit_phi_incomming> mincomming; -}; - -class -jit_terminator : public jit_instruction -{ -public: - -#define JIT_TERMINATOR_CONST(N) \ - jit_terminator (size_t asuccessor_count, \ - OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ - : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), \ - malive (asuccessor_count, false) { } - - JIT_TERMINATOR_CONST (1) - JIT_TERMINATOR_CONST (2) - JIT_TERMINATOR_CONST (3) - -#undef JIT_TERMINATOR_CONST - - jit_block *successor (size_t idx = 0) const - { - return static_cast<jit_block *> (argument (idx)); - } - - llvm::BasicBlock *successor_llvm (size_t idx = 0) const - { - return successor (idx)->to_llvm (); - } - - size_t successor_index (const jit_block *asuccessor) const; - - std::ostream& print_successor (std::ostream& os, size_t idx = 0) const - { - if (alive (idx)) - os << "[live] "; - else - os << "[dead] "; - - return successor (idx)->short_print (os); - } - - // Check if the jump to successor is live - bool alive (const jit_block *asuccessor) const - { - return alive (successor_index (asuccessor)); - } - - bool alive (size_t idx) const { return malive[idx]; } - - bool alive (int idx) const { return malive[idx]; } - - size_t successor_count (void) const { return malive.size (); } - - virtual bool infer (void); - - llvm::TerminatorInst *to_llvm (void) const; -protected: - virtual bool check_alive (size_t) const { return true; } -private: - std::vector<bool> malive; -}; - -class -jit_branch : public jit_terminator -{ -public: - jit_branch (jit_block *succ) : jit_terminator (1, succ) { } - - virtual size_t successor_count (void) const { return 1; } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent) << "branch: "; - return print_successor (os); - } - - JIT_VALUE_ACCEPT; -}; - -class -jit_cond_branch : public jit_terminator -{ -public: - jit_cond_branch (jit_value *c, jit_block *ctrue, jit_block *cfalse) - : jit_terminator (2, ctrue, cfalse, c) { } - - jit_value *cond (void) const { return argument (2); } - - std::ostream& print_cond (std::ostream& os) const - { - return cond ()->short_print (os); - } - - llvm::Value *cond_llvm (void) const - { - return cond ()->to_llvm (); - } - - virtual size_t successor_count (void) const { return 2; } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent) << "cond_branch: "; - print_cond (os) << ", "; - print_successor (os, 0) << ", "; - return print_successor (os, 1); - } - - JIT_VALUE_ACCEPT; -}; - -class -jit_call : public jit_instruction -{ -public: - jit_call (const jit_operation& (*aoperation) (void)) - : moperation (aoperation ()) - { - const jit_function& ol = overload (); - if (ol.valid ()) - stash_type (ol.result ()); - } - - jit_call (const jit_operation& aoperation) : moperation (aoperation) - { - const jit_function& ol = overload (); - if (ol.valid ()) - stash_type (ol.result ()); - } - -#define JIT_CALL_CONST(N) \ - jit_call (const jit_operation& aoperation, \ - OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ - : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), moperation (aoperation) { } \ - \ - jit_call (const jit_operation& (*aoperation) (void), \ - OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ - : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), moperation (aoperation ()) \ - { } - - JIT_CALL_CONST (1) - JIT_CALL_CONST (2) - JIT_CALL_CONST (3) - JIT_CALL_CONST (4) - -#undef JIT_CALL_CONST - - jit_call (const jit_operation& aoperation, - const std::vector<jit_value *>& args) - : jit_instruction (args), moperation (aoperation) - { } - - const jit_operation& operation (void) const { return moperation; } - - bool can_error (void) const - { - return overload ().can_error (); - } - - const jit_function& overload (void) const - { - return moperation.overload (argument_types ()); - } - - virtual bool needs_release (void) const; - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent); - - if (use_count ()) - short_print (os) << " = "; - os << "call " << moperation.name () << " ("; - - for (size_t i = 0; i < argument_count (); ++i) - { - print_argument (os, i); - if (i + 1 < argument_count ()) - os << ", "; - } - return os << ")"; - } - - virtual bool infer (void); - - JIT_VALUE_ACCEPT; -private: - const jit_operation& moperation; -}; - -// FIXME: This is just ugly... -// checks error_state, if error_state is false then goto the normal branch, -// otherwise goto the error branch -class -jit_error_check : public jit_terminator -{ -public: - // Which variable is the error check for? - enum variable - { - var_error_state, - var_interrupt - }; - - static std::string variable_to_string (variable v); - - jit_error_check (variable var, jit_call *acheck_for, jit_block *normal, - jit_block *error) - : jit_terminator (2, error, normal, acheck_for), mvariable (var) { } - - jit_error_check (variable var, jit_block *normal, jit_block *error) - : jit_terminator (2, error, normal), mvariable (var) { } - - variable check_variable (void) const { return mvariable; } - - bool has_check_for (void) const - { - return argument_count () == 3; - } - - jit_call *check_for (void) const - { - assert (has_check_for ()); - return static_cast<jit_call *> (argument (2)); - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; - - JIT_VALUE_ACCEPT; -protected: - virtual bool check_alive (size_t idx) const - { - if (! has_check_for ()) - return true; - return idx == 1 ? true : check_for ()->can_error (); - } -private: - variable mvariable; -}; - -// for now only handles the 1D case -class -jit_magic_end : public jit_instruction -{ -public: - class - context - { - public: - context (void) : value (0), index (0), count (0) - { } - - context (jit_factory& factory, jit_value *avalue, size_t aindex, - size_t acount); - - jit_value *value; - jit_const_index *index; - jit_const_index *count; - }; - - jit_magic_end (const std::vector<context>& full_context); - - virtual bool infer (void); - - const jit_function& overload () const; - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; - - context resolve_context (void) const; - - virtual std::ostream& short_print (std::ostream& os) const - { - return os << "magic_end" << "#" << id (); - } - - JIT_VALUE_ACCEPT; -private: - std::vector<context> contexts; -}; - -class -jit_extract_argument : public jit_assign_base -{ -public: - jit_extract_argument (jit_type *atype, jit_variable *adest) - : jit_assign_base (adest) - { - stash_type (atype); - } - - const std::string& name (void) const - { - return dest ()->name (); - } - - const jit_function& overload (void) const - { - return jit_typeinfo::cast (type (), jit_typeinfo::get_any ()); - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent); - - return short_print (os) << " = extract " << name (); - } - - JIT_VALUE_ACCEPT; -}; - -class -jit_store_argument : public jit_instruction -{ -public: - jit_store_argument (jit_variable *var) - : jit_instruction (var), dest (var) - { } - - const std::string& name (void) const - { - return dest->name (); - } - - const jit_function& overload (void) const - { - return jit_typeinfo::cast (jit_typeinfo::get_any (), result_type ()); - } - - jit_value *result (void) const - { - return argument (0); - } - - jit_type *result_type (void) const - { - return result ()->type (); - } - - llvm::Value *result_llvm (void) const - { - return result ()->to_llvm (); - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - jit_value *res = result (); - print_indent (os, indent) << "store "; - dest->short_print (os); - - if (! isa<jit_variable> (res)) - { - os << " = "; - res->short_print (os); - } - - return os; - } - - JIT_VALUE_ACCEPT; -private: - jit_variable *dest; -}; - -class -jit_return : public jit_instruction -{ -public: - jit_return (void) { } - - jit_return (jit_value *retval) : jit_instruction (retval) { } - - jit_value *result (void) const - { - return argument_count () ? argument (0) : 0; - } - - jit_type *result_type (void) const - { - jit_value *res = result (); - return res ? res->type () : 0; - } - - virtual std::ostream& print (std::ostream& os, size_t indent = 0) const - { - print_indent (os, indent) << "return"; - - if (result ()) - os << " " << *result (); - - return os; - } - - JIT_VALUE_ACCEPT; -}; - -class -jit_ir_walker -{ -public: - virtual ~jit_ir_walker () { } - -#define JIT_METH(clname) \ - virtual void visit (jit_ ## clname&) = 0; - - JIT_VISIT_IR_CLASSES; - -#undef JIT_METH -}; - -template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T, bool QUOTE> -void -jit_const<T, EXTRACT_T, PASS_T, QUOTE>::accept (jit_ir_walker& walker) -{ - walker.visit (*this); -} - -#undef JIT_VALUE_ACCEPT - -#endif -#endif
--- a/libinterp/corefcn/jit-typeinfo.cc Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2243 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -// defines required by llvm -#define __STDC_LIMIT_MACROS -#define __STDC_CONSTANT_MACROS - -#if defined (HAVE_CONFIG_H) -# include "config.h" -#endif - -#if defined (HAVE_LLVM) - -#include "jit-typeinfo.h" - -#if defined (HAVE_LLVM_IR_VERIFIER_H) -# include <llvm/IR/Verifier.h> -#else -# include <llvm/Analysis/Verifier.h> -#endif - -#include <llvm/ExecutionEngine/ExecutionEngine.h> - -#if defined (HAVE_LLVM_IR_FUNCTION_H) -# include <llvm/IR/GlobalVariable.h> -# include <llvm/IR/LLVMContext.h> -# include <llvm/IR/Function.h> -# include <llvm/IR/Instructions.h> -# include <llvm/IR/Intrinsics.h> -#else -# include <llvm/GlobalVariable.h> -# include <llvm/LLVMContext.h> -# include <llvm/Function.h> -# include <llvm/Instructions.h> -# include <llvm/Intrinsics.h> -#endif - -#if defined (HAVE_LLVM_SUPPORT_IRBUILDER_H) -# include <llvm/Support/IRBuilder.h> -# elif defined(HAVE_LLVM_IR_IRBUILDER_H) -# include <llvm/IR/IRBuilder.h> -#else -# include <llvm/IRBuilder.h> -#endif - -#include <llvm/Support/raw_os_ostream.h> - -#include "jit-ir.h" -#include "ov.h" -#include "ov-builtin.h" -#include "ov-complex.h" -#include "ov-scalar.h" -#include "pager.h" - -static llvm::LLVMContext& context = llvm::getGlobalContext (); - -jit_typeinfo *jit_typeinfo::instance = 0; - -std::ostream& jit_print (std::ostream& os, jit_type *atype) -{ - if (! atype) - return os << "null"; - return os << atype->name (); -} - -// function that jit code calls -extern "C" void -octave_jit_print_any (const char *name, octave_base_value *obv) -{ - obv->print_with_name (octave_stdout, name, true); -} - -extern "C" void -octave_jit_print_scalar (const char *name, double value) -{ - // FIXME: We should avoid allocating a new octave_scalar each time - octave_value ov (value); - ov.print_with_name (octave_stdout, name); -} - -extern "C" octave_base_value* -octave_jit_binary_any_any (octave_value::binary_op op, octave_base_value *lhs, - octave_base_value *rhs) -{ - octave_value olhs (lhs, true); - octave_value orhs (rhs, true); - octave_value result = do_binary_op (op, olhs, orhs); - octave_base_value *rep = result.internal_rep (); - rep->grab (); - return rep; -} - -extern "C" octave_idx_type -octave_jit_compute_nelem (double base, double limit, double inc) -{ - Range rng = Range (base, limit, inc); - return rng.numel (); -} - -extern "C" void -octave_jit_release_any (octave_base_value *obv) -{ - obv->release (); -} - -extern "C" void -octave_jit_release_matrix (jit_matrix *m) -{ - delete m->array; -} - -extern "C" octave_base_value * -octave_jit_grab_any (octave_base_value *obv) -{ - obv->grab (); - return obv; -} - -extern "C" jit_matrix -octave_jit_grab_matrix (jit_matrix *m) -{ - return *m->array; -} - -extern "C" octave_base_value * -octave_jit_cast_any_matrix (jit_matrix *m) -{ - octave_value ret (*m->array); - octave_base_value *rep = ret.internal_rep (); - rep->grab (); - delete m->array; - - return rep; -} - -extern "C" jit_matrix -octave_jit_cast_matrix_any (octave_base_value *obv) -{ - NDArray m = obv->array_value (); - obv->release (); - return m; -} - -extern "C" octave_base_value * -octave_jit_cast_any_range (jit_range *rng) -{ - Range temp (*rng); - octave_value ret (temp); - octave_base_value *rep = ret.internal_rep (); - rep->grab (); - - return rep; -} -extern "C" jit_range -octave_jit_cast_range_any (octave_base_value *obv) -{ - - jit_range r (obv->range_value ()); - obv->release (); - return r; -} - -extern "C" double -octave_jit_cast_scalar_any (octave_base_value *obv) -{ - double ret = obv->double_value (); - obv->release (); - return ret; -} - -extern "C" octave_base_value * -octave_jit_cast_any_scalar (double value) -{ - return new octave_scalar (value); -} - -extern "C" Complex -octave_jit_cast_complex_any (octave_base_value *obv) -{ - Complex ret = obv->complex_value (); - obv->release (); - return ret; -} - -extern "C" octave_base_value * -octave_jit_cast_any_complex (Complex c) -{ - if (c.imag () == 0) - return new octave_scalar (c.real ()); - else - return new octave_complex (c); -} - -extern "C" void -octave_jit_octave::err_nan_to_logical_conversion (void) -{ - octave::err_nan_to_logical_conversion (); -} - -extern "C" void -octave_jit_ginvalid_index (void) -{ - // FIXME: 0-argument form of octave::err_invalid_index removed in cset dd6345fd8a97 - // Report -1 as the bad index for all occurrences. - octave::err_invalid_index (static_cast<octave_idx_type> (-1)); -} - -extern "C" void -octave_jit_gindex_range (int nd, int dim, octave_idx_type iext, - octave_idx_type ext) -{ - octave::err_index_out_of_range (nd, dim, iext, ext); -} - -extern "C" jit_matrix -octave_jit_paren_subsasgn_impl (jit_matrix *mat, octave_idx_type index, - double value) -{ - NDArray *array = mat->array; - if (array->numel () < index) - array->resize1 (index); - - double *data = array->fortran_vec (); - data[index - 1] = value; - - mat->update (); - return *mat; -} - -static void -make_indices (double *indices, octave_idx_type idx_count, - Array<idx_vector>& result) -{ - result.resize (dim_vector (1, idx_count)); - for (octave_idx_type i = 0; i < idx_count; ++i) - result(i) = idx_vector (indices[i]); -} - -extern "C" double -octave_jit_paren_scalar (jit_matrix *mat, double *indicies, - octave_idx_type idx_count) -{ - // FIXME: Replace this with a more optimal version - Array<idx_vector> idx; - make_indices (indicies, idx_count, idx); - - Array<double> ret = mat->array->index (idx); - - return ret.xelem (0); -} - -extern "C" jit_matrix -octave_jit_paren_scalar_subsasgn (jit_matrix *mat, double *indices, - octave_idx_type idx_count, double value) -{ - // FIXME: Replace this with a more optimal version - jit_matrix ret; - - Array<idx_vector> idx; - make_indices (indices, idx_count, idx); - - Matrix temp (1, 1); - temp.xelem(0) = value; - mat->array->assign (idx, temp); - ret.update (mat->array); - - return ret; -} - -extern "C" jit_matrix -octave_jit_paren_subsasgn_matrix_range (jit_matrix *mat, jit_range *index, - double value) -{ - NDArray *array = mat->array; - bool done = false; - - // optimize for the simple case (no resizing and no errors) - if (*array->jit_ref_count () == 1 - && index->all_elements_are_ints ()) - { - // this code is similar to idx_vector::fill, but we avoid allocating an - // idx_vector and its associated rep - octave_idx_type start = static_cast<octave_idx_type> (index->base) - 1; - octave_idx_type step = static_cast<octave_idx_type> (index->inc); - octave_idx_type nelem = index->nelem; - octave_idx_type final = start + nelem * step; - if (step < 0) - { - step = -step; - std::swap (final, start); - } - - if (start >= 0 && final < mat->slice_len) - { - done = true; - - double *data = array->jit_slice_data (); - if (step == 1) - std::fill (data + start, data + start + nelem, value); - else - { - for (octave_idx_type i = start; i < final; i += step) - data[i] = value; - } - } - } - - if (! done) - { - idx_vector idx (*index); - NDArray avalue (dim_vector (1, 1)); - avalue.xelem (0) = value; - array->assign (idx, avalue); - } - - jit_matrix ret; - ret.update (array); - return ret; -} - -extern "C" double -octave_jit_end_matrix (jit_matrix *mat, octave_idx_type idx, - octave_idx_type count) -{ - octave_idx_type ndim = mat->dimensions[-1]; - if (ndim == count) - return mat->dimensions[idx]; - else if (ndim > count) - { - if (idx == count - 1) - { - double ret = mat->dimensions[idx]; - for (octave_idx_type i = idx + 1; i < ndim; ++i) - ret *= mat->dimensions[idx]; - return ret; - } - - return mat->dimensions[idx]; - } - else // ndim < count - return idx < ndim ? mat->dimensions[idx] : 1; -} - -extern "C" octave_base_value * -octave_jit_create_undef (void) -{ - octave_value undef; - octave_base_value *ret = undef.internal_rep (); - ret->grab (); - - return ret; -} - -extern "C" Complex -octave_jit_complex_mul (Complex lhs, Complex rhs) -{ - if (lhs.imag () == 0 && rhs.imag() == 0) - return Complex (lhs.real () * rhs.real (), 0); - - return lhs * rhs; -} - -extern "C" Complex -octave_jit_complex_div (Complex lhs, Complex rhs) -{ - // see src/OPERATORS/op-cs-cs.cc - if (rhs == 0.0) - warn_divide_by_zero (); - - return lhs / rhs; -} - -// FIXME: CP form src/xpow.cc -static inline int -xisint (double x) -{ - return (octave::math::x_nint (x) == x - && ((x >= 0 && x < std::numeric_limits<int>::max ()) - || (x <= 0 && x > std::numeric_limits<int>::min ()))); -} - -extern "C" Complex -octave_jit_pow_scalar_scalar (double lhs, double rhs) -{ - // FIXME: almost CP from src/xpow.cc - if (lhs < 0.0 && ! xisint (rhs)) - return std::pow (Complex (lhs), rhs); - return std::pow (lhs, rhs); -} - -extern "C" Complex -octave_jit_pow_complex_complex (Complex lhs, Complex rhs) -{ - if (lhs.imag () == 0 && rhs.imag () == 0) - return octave_jit_pow_scalar_scalar (lhs.real (), rhs.real ()); - return std::pow (lhs, rhs); -} - -extern "C" Complex -octave_jit_pow_complex_scalar (Complex lhs, double rhs) -{ - if (lhs.imag () == 0) - return octave_jit_pow_scalar_scalar (lhs.real (), rhs); - return std::pow (lhs, rhs); -} - -extern "C" Complex -octave_jit_pow_scalar_complex (double lhs, Complex rhs) -{ - if (rhs.imag () == 0) - return octave_jit_pow_scalar_scalar (lhs, rhs.real ()); - return std::pow (lhs, rhs); -} - -extern "C" void -octave_jit_print_matrix (jit_matrix *m) -{ - std::cout << *m << std::endl; -} - -OCTAVE_NORETURN static -void -err_bad_result (void) -{ - error ("incorrect type information given to the JIT compiler"); -} - -// FIXME: Add support for multiple outputs -extern "C" octave_base_value * -octave_jit_call (octave_builtin::fcn fn, size_t nargin, - octave_base_value **argin, jit_type *result_type) -{ - octave_value_list ovl (nargin); - for (size_t i = 0; i < nargin; ++i) - ovl.xelem (i) = octave_value (argin[i]); - - ovl = fn (ovl, 1); - - // FIXME: Check result_type somehow - if (result_type) - { - if (ovl.length () < 1) - err_bad_result (); - - octave_value result = ovl.xelem(0); - octave_base_value *ret = result.internal_rep (); - ret->grab (); - return ret; - } - - if (! (ovl.empty () - || (ovl.length () == 1 && ovl.xelem (0).is_undefined ()))) - err_bad_result (); - - return 0; -} - -// -------------------- jit_range -------------------- -bool -jit_range::all_elements_are_ints () const -{ - Range r (*this); - return r.all_elements_are_ints (); -} - -std::ostream& -operator<< (std::ostream& os, const jit_range& rng) -{ - return os << "Range[" << rng.base << ", " << rng.limit << ", " << rng.inc - << ", " << rng.nelem << "]"; -} - -// -------------------- jit_matrix -------------------- - -std::ostream& -operator<< (std::ostream& os, const jit_matrix& mat) -{ - return os << "Matrix[" << mat.ref_count << ", " << mat.slice_data << ", " - << mat.slice_len << ", " << mat.dimensions << ", " - << mat.array << "]"; -} - -// -------------------- jit_type -------------------- -jit_type::jit_type (const std::string& aname, jit_type *aparent, - llvm::Type *allvm_type, bool askip_paren, int aid) : - mname (aname), mparent (aparent), llvm_type (allvm_type), mid (aid), - mdepth (aparent ? aparent->mdepth + 1 : 0), mskip_paren (askip_paren) -{ - std::memset (msret, 0, sizeof (msret)); - std::memset (mpointer_arg, 0, sizeof (mpointer_arg)); - std::memset (mpack, 0, sizeof (mpack)); - std::memset (munpack, 0, sizeof (munpack)); - - for (size_t i = 0; i < jit_convention::length; ++i) - mpacked_type[i] = llvm_type; -} - -llvm::Type * -jit_type::to_llvm_arg (void) const -{ - return llvm_type ? llvm_type->getPointerTo () : 0; -} - -// -------------------- jit_function -------------------- -jit_function::jit_function () : module (0), llvm_function (0), mresult (0), - call_conv (jit_convention::length), - mcan_error (false) -{ } - -jit_function::jit_function (llvm::Module *amodule, - jit_convention::type acall_conv, - const llvm::Twine& aname, jit_type *aresult, - const std::vector<jit_type *>& aargs) - : module (amodule), mresult (aresult), args (aargs), call_conv (acall_conv), - mcan_error (false) -{ - llvm::SmallVector<llvm::Type *, 15> llvm_args; - - llvm::Type *rtype = llvm::Type::getVoidTy (context); - if (mresult) - { - rtype = mresult->packed_type (call_conv); - if (sret ()) - { - llvm_args.push_back (rtype->getPointerTo ()); - rtype = llvm::Type::getVoidTy (context); - } - } - - for (std::vector<jit_type *>::const_iterator iter = args.begin (); - iter != args.end (); ++iter) - { - jit_type *ty = *iter; - assert (ty); - llvm::Type *argty = ty->packed_type (call_conv); - if (ty->pointer_arg (call_conv)) - argty = argty->getPointerTo (); - - llvm_args.push_back (argty); - } - - // we mark all functinos as external linkage because this prevents llvm - // from getting rid of always inline functions - llvm::FunctionType *ft = llvm::FunctionType::get (rtype, llvm_args, false); - llvm_function = llvm::Function::Create (ft, llvm::Function::ExternalLinkage, - aname, module); - - if (sret ()) - { -#if defined (FUNCTION_ADDATTRIBUTE_ARG_IS_ATTRIBUTES) - llvm::AttrBuilder attr_builder; - attr_builder.addAttribute (llvm::Attributes::StructRet); - llvm::Attributes attrs = llvm::Attributes::get(context, attr_builder); - llvm_function->addAttribute (1, attrs); -#else - llvm_function->addAttribute (1, llvm::Attribute::StructRet); -#endif - } - - if (call_conv == jit_convention::internal) -#if defined (FUNCTION_ADDFNATTR_ARG_IS_ATTRIBUTES) - llvm_function->addFnAttr (llvm::Attributes::AlwaysInline); -#else - llvm_function->addFnAttr (llvm::Attribute::AlwaysInline); -#endif -} - -jit_function::jit_function (const jit_function& fn, jit_type *aresult, - const std::vector<jit_type *>& aargs) - : module (fn.module), llvm_function (fn.llvm_function), mresult (aresult), - args (aargs), call_conv (fn.call_conv), mcan_error (fn.mcan_error) -{ } - -jit_function::jit_function (const jit_function& fn) - : module (fn.module), llvm_function (fn.llvm_function), mresult (fn.mresult), - args (fn.args), call_conv (fn.call_conv), mcan_error (fn.mcan_error) -{ } - -void -jit_function::erase (void) -{ - if (! llvm_function) - return; - - llvm_function->eraseFromParent (); - llvm_function = 0; -} - -std::string -jit_function::name (void) const -{ - return llvm_function->getName (); -} - -llvm::BasicBlock * -jit_function::new_block (const std::string& aname, - llvm::BasicBlock *insert_before) -{ - return llvm::BasicBlock::Create (context, aname, llvm_function, - insert_before); -} - -llvm::Value * -jit_function::call (llvm::IRBuilderD& builder, - const std::vector<jit_value *>& in_args) const -{ - if (! valid ()) - throw jit_fail_exception ("Call not implemented"); - - assert (in_args.size () == args.size ()); - std::vector<llvm::Value *> llvm_args (args.size ()); - for (size_t i = 0; i < in_args.size (); ++i) - llvm_args[i] = in_args[i]->to_llvm (); - - return call (builder, llvm_args); -} - -llvm::Value * -jit_function::call (llvm::IRBuilderD& builder, - const std::vector<llvm::Value *>& in_args) const -{ - if (! valid ()) - throw jit_fail_exception ("Call not implemented"); - - assert (in_args.size () == args.size ()); - llvm::SmallVector<llvm::Value *, 10> llvm_args; - llvm_args.reserve (in_args.size () + sret ()); - - llvm::BasicBlock *insert_block = builder.GetInsertBlock (); - llvm::Function *parent = insert_block->getParent (); - assert (parent); - - // we insert allocas inside the prelude block to prevent stack overflows - llvm::BasicBlock& prelude = parent->getEntryBlock (); - llvm::IRBuilder<> pre_builder (&prelude, prelude.begin ()); - - llvm::AllocaInst *sret_mem = 0; - if (sret ()) - { - sret_mem = pre_builder.CreateAlloca (mresult->packed_type (call_conv)); - llvm_args.push_back (sret_mem); - } - - for (size_t i = 0; i < in_args.size (); ++i) - { - llvm::Value *arg = in_args[i]; - jit_type::convert_fn convert = args[i]->pack (call_conv); - if (convert) - arg = convert (builder, arg); - - if (args[i]->pointer_arg (call_conv)) - { - llvm::Type *ty = args[i]->packed_type (call_conv); - llvm::Value *alloca = pre_builder.CreateAlloca (ty); - builder.CreateStore (arg, alloca); - arg = alloca; - } - - llvm_args.push_back (arg); - } - - llvm::CallInst *callinst = builder.CreateCall (llvm_function, llvm_args); - llvm::Value *ret = callinst; - - if (sret ()) - { -#if defined (CALLINST_ADDATTRIBUTE_ARG_IS_ATTRIBUTES) - llvm::AttrBuilder attr_builder; - attr_builder.addAttribute(llvm::Attributes::StructRet); - llvm::Attributes attrs = llvm::Attributes::get(context, attr_builder); - callinst->addAttribute (1, attrs); -#else - callinst->addAttribute (1, llvm::Attribute::StructRet); -#endif - ret = builder.CreateLoad (sret_mem); - } - - if (mresult) - { - jit_type::convert_fn unpack = mresult->unpack (call_conv); - if (unpack) - ret = unpack (builder, ret); - } - - return ret; -} - -llvm::Value * -jit_function::argument (llvm::IRBuilderD& builder, size_t idx) const -{ - assert (idx < args.size ()); - - // FIXME: We should be treating arguments like a list, not a vector. - // Shouldn't matter much for now, as the number of arguments shouldn't - // be much bigger than 4 - llvm::Function::arg_iterator iter = llvm_function->arg_begin (); - if (sret ()) - ++iter; - - for (size_t i = 0; i < idx; ++i, ++iter); - - if (args[idx]->pointer_arg (call_conv)) - return builder.CreateLoad (iter); - - return iter; -} - -void -jit_function::do_return (llvm::IRBuilderD& builder, llvm::Value *rval, - bool verify) -{ - assert (! rval == ! mresult); - - if (rval) - { - jit_type::convert_fn convert = mresult->pack (call_conv); - if (convert) - rval = convert (builder, rval); - - if (sret ()) - { - builder.CreateStore (rval, llvm_function->arg_begin ()); - builder.CreateRetVoid (); - } - else - builder.CreateRet (rval); - } - else - builder.CreateRetVoid (); - - if (verify) - llvm::verifyFunction (*llvm_function); -} - -void -jit_function::do_add_mapping (llvm::ExecutionEngine *engine, void *fn) -{ - assert (valid ()); - engine->addGlobalMapping (llvm_function, fn); -} - -std::ostream& -operator<< (std::ostream& os, const jit_function& fn) -{ - llvm::Function *lfn = fn.to_llvm (); - os << "jit_function: cc=" << fn.call_conv; - llvm::raw_os_ostream llvm_out (os); - lfn->print (llvm_out); - llvm_out.flush (); - return os; -} - -// -------------------- jit_operation -------------------- -jit_operation::~jit_operation (void) -{ - for (generated_map::iterator iter = generated.begin (); - iter != generated.end (); ++iter) - { - delete iter->first; - delete iter->second; - } -} - -void -jit_operation::add_overload (const jit_function& func, - const std::vector<jit_type*>& args) -{ - if (args.size () >= overloads.size ()) - overloads.resize (args.size () + 1); - - Array<jit_function>& over = overloads[args.size ()]; - dim_vector dv (over.dims ()); - Array<octave_idx_type> idx = to_idx (args); - bool must_resize = false; - - if (dv.length () != idx.numel ()) - { - dv.resize (idx.numel ()); - must_resize = true; - } - - for (octave_idx_type i = 0; i < dv.length (); ++i) - if (dv(i) <= idx(i)) - { - must_resize = true; - dv(i) = idx(i) + 1; - } - - if (must_resize) - over.resize (dv); - - over(idx) = func; -} - -const jit_function& -jit_operation::overload (const std::vector<jit_type*>& types) const -{ - static jit_function null_overload; - for (size_t i = 0; i < types.size (); ++i) - if (! types[i]) - return null_overload; - - if (types.size () >= overloads.size ()) - return do_generate (types); - - const Array<jit_function>& over = overloads[types.size ()]; - dim_vector dv (over.dims ()); - Array<octave_idx_type> idx = to_idx (types); - for (octave_idx_type i = 0; i < dv.length (); ++i) - if (idx(i) >= dv(i)) - return do_generate (types); - - const jit_function& ret = over(idx); - if (! ret.valid ()) - return do_generate (types); - - return ret; -} - -Array<octave_idx_type> -jit_operation::to_idx (const std::vector<jit_type*>& types) const -{ - octave_idx_type numel = types.size (); - numel = std::max (numel, static_cast<octave_idx_type>(2)); - - Array<octave_idx_type> idx (dim_vector (1, numel)); - for (octave_idx_type i = 0; i < static_cast<octave_idx_type> (types.size ()); - ++i) - idx(i) = types[i]->type_id (); - - if (types.size () == 0) - idx(0) = idx(1) = 0; - if (types.size () == 1) - { - idx(1) = idx(0); - idx(0) = 0; - } - - return idx; -} - -const jit_function& -jit_operation::do_generate (const signature_vec& types) const -{ - static jit_function null_overload; - generated_map::const_iterator find = generated.find (&types); - if (find != generated.end ()) - { - if (find->second) - return *find->second; - else - return null_overload; - } - - jit_function *ret = generate (types); - generated[new signature_vec (types)] = ret; - return ret ? *ret : null_overload; -} - -jit_function * -jit_operation::generate (const signature_vec&) const -{ - return 0; -} - -bool -jit_operation::signature_cmp::operator () (const signature_vec *lhs, - const signature_vec *rhs) const -{ - const signature_vec& l = *lhs; - const signature_vec& r = *rhs; - - if (l.size () < r.size ()) - return true; - else if (l.size () > r.size ()) - return false; - - for (size_t i = 0; i < l.size (); ++i) - { - if (l[i]->type_id () < r[i]->type_id ()) - return true; - else if (l[i]->type_id () > r[i]->type_id ()) - return false; - } - - return false; -} - -// -------------------- jit_index_operation -------------------- -jit_function * -jit_index_operation::generate (const signature_vec& types) const -{ - if (types.size () > 2 && types[0] == jit_typeinfo::get_matrix ()) - { - // indexing a matrix with scalars - jit_type *scalar = jit_typeinfo::get_scalar (); - for (size_t i = 1; i < types.size (); ++i) - if (types[i] != scalar) - return 0; - - return generate_matrix (types); - } - - return 0; -} - -llvm::Value * -jit_index_operation::create_arg_array (llvm::IRBuilderD& builder, - const jit_function &fn, size_t start_idx, - size_t end_idx) const -{ - size_t n = end_idx - start_idx; - llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); - llvm::ArrayType *array_t = llvm::ArrayType::get (scalar_t, n); - llvm::Value *array = llvm::UndefValue::get (array_t); - for (size_t i = start_idx; i < end_idx; ++i) - { - llvm::Value *idx = fn.argument (builder, i); - array = builder.CreateInsertValue (array, idx, i - start_idx); - } - - llvm::Value *array_mem = builder.CreateAlloca (array_t); - builder.CreateStore (array, array_mem); - return builder.CreateBitCast (array_mem, scalar_t->getPointerTo ()); -} - -// -------------------- jit_paren_subsref -------------------- -jit_function * -jit_paren_subsref::generate_matrix (const signature_vec& types) const -{ - std::stringstream ss; - ss << "jit_paren_subsref_matrix_scalar" << (types.size () - 1); - - jit_type *scalar = jit_typeinfo::get_scalar (); - jit_function *fn = new jit_function (module, jit_convention::internal, - ss.str (), scalar, types); - fn->mark_can_error (); - llvm::BasicBlock *body = fn->new_block (); - llvm::IRBuilder<> builder (body); - - llvm::Value *array = create_arg_array (builder, *fn, 1, types.size ()); - jit_type *index = jit_typeinfo::get_index (); - llvm::Value *nelem = llvm::ConstantInt::get (index->to_llvm (), - types.size () - 1); - llvm::Value *mat = fn->argument (builder, 0); - llvm::Value *ret = paren_scalar.call (builder, mat, array, nelem); - fn->do_return (builder, ret); - return fn; -} - -void -jit_paren_subsref::do_initialize (void) -{ - std::vector<jit_type *> types (3); - types[0] = jit_typeinfo::get_matrix (); - types[1] = jit_typeinfo::get_scalar_ptr (); - types[2] = jit_typeinfo::get_index (); - - jit_type *scalar = jit_typeinfo::get_scalar (); - paren_scalar = jit_function (module, jit_convention::external, - "octave_jit_paren_scalar", scalar, types); - paren_scalar.add_mapping (engine, &octave_jit_paren_scalar); - paren_scalar.mark_can_error (); -} - -// -------------------- jit_paren_subsasgn -------------------- -jit_function * -jit_paren_subsasgn::generate_matrix (const signature_vec& types) const -{ - std::stringstream ss; - ss << "jit_paren_subsasgn_matrix_scalar" << (types.size () - 2); - - jit_type *matrix = jit_typeinfo::get_matrix (); - jit_function *fn = new jit_function (module, jit_convention::internal, - ss.str (), matrix, types); - fn->mark_can_error (); - llvm::BasicBlock *body = fn->new_block (); - llvm::IRBuilder<> builder (body); - - llvm::Value *array = create_arg_array (builder, *fn, 1, types.size () - 1); - jit_type *index = jit_typeinfo::get_index (); - llvm::Value *nelem = llvm::ConstantInt::get (index->to_llvm (), - types.size () - 2); - - llvm::Value *mat = fn->argument (builder, 0); - llvm::Value *value = fn->argument (builder, types.size () - 1); - llvm::Value *ret = paren_scalar.call (builder, mat, array, nelem, value); - fn->do_return (builder, ret); - return fn; -} - -void -jit_paren_subsasgn::do_initialize (void) -{ - if (paren_scalar.valid ()) - return; - - jit_type *matrix = jit_typeinfo::get_matrix (); - std::vector<jit_type *> types (4); - types[0] = matrix; - types[1] = jit_typeinfo::get_scalar_ptr (); - types[2] = jit_typeinfo::get_index (); - types[3] = jit_typeinfo::get_scalar (); - - paren_scalar = jit_function (module, jit_convention::external, - "octave_jit_paren_scalar", matrix, types); - paren_scalar.add_mapping (engine, &octave_jit_paren_scalar_subsasgn); - paren_scalar.mark_can_error (); -} - -// -------------------- jit_typeinfo -------------------- -void -jit_typeinfo::initialize (llvm::Module *m, llvm::ExecutionEngine *e) -{ - new jit_typeinfo (m, e); -} - -// wrap function names to simplify jit_typeinfo::create_external -#define JIT_FN(fn) engine, &fn, #fn - -jit_typeinfo::jit_typeinfo (llvm::Module *m, llvm::ExecutionEngine *e) - : module (m), engine (e), next_id (0), - builder (*new llvm::IRBuilderD (context)) -{ - instance = this; - - // FIXME: We should be registering types like in octave_value_typeinfo - llvm::Type *any_t = llvm::StructType::create (context, "octave_base_value"); - any_t = any_t->getPointerTo (); - - llvm::Type *scalar_t = llvm::Type::getDoubleTy (context); - llvm::Type *bool_t = llvm::Type::getInt1Ty (context); - llvm::Type *string_t = llvm::Type::getInt8Ty (context); - string_t = string_t->getPointerTo (); - llvm::Type *index_t = llvm::Type::getIntNTy (context, - sizeof(octave_idx_type) * 8); - - llvm::StructType *range_t = llvm::StructType::create (context, "range"); - std::vector<llvm::Type *> range_contents (4, scalar_t); - range_contents[3] = index_t; - range_t->setBody (range_contents); - - llvm::Type *refcount_t = llvm::Type::getIntNTy (context, sizeof(int) * 8); - - llvm::StructType *matrix_t = llvm::StructType::create (context, "matrix"); - llvm::Type *matrix_contents[5]; - matrix_contents[0] = refcount_t->getPointerTo (); - matrix_contents[1] = scalar_t->getPointerTo (); - matrix_contents[2] = index_t; - matrix_contents[3] = index_t->getPointerTo (); - matrix_contents[4] = string_t; - matrix_t->setBody (llvm::makeArrayRef (matrix_contents, 5)); - - llvm::Type *complex_t = llvm::ArrayType::get (scalar_t, 2); - - // complex_ret is what is passed to C functions in order to get calling - // convention right - llvm::Type *cmplx_inner_cont[] = {scalar_t, scalar_t}; - llvm::StructType *cmplx_inner = llvm::StructType::create (cmplx_inner_cont); - - complex_ret = llvm::StructType::create (context, "complex_ret"); - { - llvm::Type *contents[] = {cmplx_inner}; - complex_ret->setBody (contents); - } - - // create types - any = new_type ("any", 0, any_t); - matrix = new_type ("matrix", any, matrix_t); - complex = new_type ("complex", any, complex_t); - scalar = new_type ("scalar", complex, scalar_t); - scalar_ptr = new_type ("scalar_ptr", 0, scalar_t->getPointerTo ()); - any_ptr = new_type ("any_ptr", 0, any_t->getPointerTo ()); - range = new_type ("range", any, range_t); - string = new_type ("string", any, string_t); - boolean = new_type ("bool", any, bool_t); - index = new_type ("index", any, index_t); - - create_int (8); - create_int (16); - create_int (32); - create_int (64); - - casts.resize (next_id + 1); - identities.resize (next_id + 1); - - // specify calling conventions - // FIXME: We should detect architecture and do something sane based on that - // here we assume x86 or x86_64 - matrix->mark_sret (jit_convention::external); - matrix->mark_pointer_arg (jit_convention::external); - - range->mark_sret (jit_convention::external); - range->mark_pointer_arg (jit_convention::external); - - complex->set_pack (jit_convention::external, &jit_typeinfo::pack_complex); - complex->set_unpack (jit_convention::external, &jit_typeinfo::unpack_complex); - complex->set_packed_type (jit_convention::external, complex_ret); - - if (sizeof (void *) == 4) - complex->mark_sret (jit_convention::external); - - paren_subsref_fn.initialize (module, engine); - paren_subsasgn_fn.initialize (module, engine); - - // bind global variables - lerror_state = new llvm::GlobalVariable (*module, bool_t, false, - llvm::GlobalValue::ExternalLinkage, - 0, "error_state"); - engine->addGlobalMapping (lerror_state, - reinterpret_cast<void *> (&error_state)); - - // sig_atomic_type is going to be some sort of integer - sig_atomic_type = llvm::Type::getIntNTy (context, sizeof(sig_atomic_t) * 8); - loctave_interrupt_state - = new llvm::GlobalVariable (*module, sig_atomic_type, false, - llvm::GlobalValue::ExternalLinkage, 0, - "octave_interrupt_state"); - engine->addGlobalMapping (loctave_interrupt_state, - reinterpret_cast<void *> (&octave_interrupt_state)); - - // generic call function - { - jit_type *int_t = intN (sizeof (octave_builtin::fcn) * 8); - any_call = create_external (JIT_FN (octave_jit_call), any, int_t, int_t, - any_ptr, int_t); - } - - // any with anything is an any op - jit_function fn; - jit_type *binary_op_type = intN (sizeof (octave_value::binary_op) * 8); - llvm::Type *llvm_bo_type = binary_op_type->to_llvm (); - jit_function any_binary = create_external (JIT_FN (octave_jit_binary_any_any), - any, binary_op_type, any, any); - any_binary.mark_can_error (); - binary_ops.resize (octave_value::num_binary_ops); - for (size_t i = 0; i < octave_value::num_binary_ops; ++i) - { - octave_value::binary_op op = static_cast<octave_value::binary_op> (i); - std::string op_name = octave_value::binary_op_as_string (op); - binary_ops[i].stash_name ("binary" + op_name); - } - - unary_ops.resize (octave_value::num_unary_ops); - for (size_t i = 0; i < octave_value::num_unary_ops; ++i) - { - octave_value::unary_op op = static_cast<octave_value::unary_op> (i); - std::string op_name = octave_value::unary_op_as_string (op); - unary_ops[i].stash_name ("unary" + op_name); - } - - for (int op = 0; op < octave_value::num_binary_ops; ++op) - { - llvm::Twine fn_name ("octave_jit_binary_any_any_"); - fn_name = fn_name + llvm::Twine (op); - - fn = create_internal (fn_name, any, any, any); - fn.mark_can_error (); - llvm::BasicBlock *block = fn.new_block (); - builder.SetInsertPoint (block); - llvm::APInt op_int(sizeof (octave_value::binary_op) * 8, op, - std::numeric_limits<octave_value::binary_op>::is_signed); - llvm::Value *op_as_llvm = llvm::ConstantInt::get (llvm_bo_type, op_int); - llvm::Value *ret = any_binary.call (builder, op_as_llvm, - fn.argument (builder, 0), - fn.argument (builder, 1)); - fn.do_return (builder, ret); - binary_ops[op].add_overload (fn); - } - - // grab matrix - fn = create_external (JIT_FN (octave_jit_grab_matrix), matrix, matrix); - grab_fn.add_overload (fn); - - grab_fn.add_overload (create_identity (scalar)); - grab_fn.add_overload (create_identity (scalar_ptr)); - grab_fn.add_overload (create_identity (any_ptr)); - grab_fn.add_overload (create_identity (boolean)); - grab_fn.add_overload (create_identity (complex)); - grab_fn.add_overload (create_identity (index)); - - // release any - fn = create_external (JIT_FN (octave_jit_release_any), 0, any); - release_fn.add_overload (fn); - release_fn.stash_name ("release"); - - // release matrix - fn = create_external (JIT_FN (octave_jit_release_matrix), 0, matrix); - release_fn.add_overload (fn); - - // destroy - destroy_fn = release_fn; - destroy_fn.stash_name ("destroy"); - destroy_fn.add_overload (create_identity(scalar)); - destroy_fn.add_overload (create_identity(boolean)); - destroy_fn.add_overload (create_identity(index)); - destroy_fn.add_overload (create_identity(complex)); - - // -------------------- scalar related operations -------------------- - - // now for binary scalar operations - add_binary_op (scalar, octave_value::op_add, llvm::Instruction::FAdd); - add_binary_op (scalar, octave_value::op_sub, llvm::Instruction::FSub); - add_binary_op (scalar, octave_value::op_mul, llvm::Instruction::FMul); - add_binary_op (scalar, octave_value::op_el_mul, llvm::Instruction::FMul); - - add_binary_fcmp (scalar, octave_value::op_lt, llvm::CmpInst::FCMP_ULT); - add_binary_fcmp (scalar, octave_value::op_le, llvm::CmpInst::FCMP_ULE); - add_binary_fcmp (scalar, octave_value::op_eq, llvm::CmpInst::FCMP_UEQ); - add_binary_fcmp (scalar, octave_value::op_ge, llvm::CmpInst::FCMP_UGE); - add_binary_fcmp (scalar, octave_value::op_gt, llvm::CmpInst::FCMP_UGT); - add_binary_fcmp (scalar, octave_value::op_ne, llvm::CmpInst::FCMP_UNE); - - jit_function gripe_div0 = create_external (JIT_FN (warn_divide_by_zero), 0); - gripe_div0.mark_can_error (); - - // divide is annoying because it might error - fn = create_internal ("octave_jit_div_scalar_scalar", scalar, scalar, scalar); - fn.mark_can_error (); - - llvm::BasicBlock *body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::BasicBlock *warn_block = fn.new_block ("warn"); - llvm::BasicBlock *normal_block = fn.new_block ("normal"); - - llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); - llvm::Value *check = builder.CreateFCmpUEQ (zero, fn.argument (builder, 1)); - builder.CreateCondBr (check, warn_block, normal_block); - - builder.SetInsertPoint (warn_block); - gripe_div0.call (builder); - builder.CreateBr (normal_block); - - builder.SetInsertPoint (normal_block); - llvm::Value *ret = builder.CreateFDiv (fn.argument (builder, 0), - fn.argument (builder, 1)); - fn.do_return (builder, ret); - } - binary_ops[octave_value::op_div].add_overload (fn); - binary_ops[octave_value::op_el_div].add_overload (fn); - - // ldiv is the same as div with the operators reversed - fn = mirror_binary (fn); - binary_ops[octave_value::op_ldiv].add_overload (fn); - binary_ops[octave_value::op_el_ldiv].add_overload (fn); - - // In general, the result of scalar ^ scalar is a complex number. We might - // be able to improve on this if we keep track of the range of values - // variables can take on. - fn = create_external (JIT_FN (octave_jit_pow_scalar_scalar), complex, scalar, - scalar); - binary_ops[octave_value::op_pow].add_overload (fn); - binary_ops[octave_value::op_el_pow].add_overload (fn); - - // now for unary scalar operations - // FIXME: Impelment not - fn = create_internal ("octave_jit_++", scalar, scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *one = llvm::ConstantFP::get (scalar_t, 1); - llvm::Value *val = fn.argument (builder, 0); - val = builder.CreateFAdd (val, one); - fn.do_return (builder, val); - } - unary_ops[octave_value::op_incr].add_overload (fn); - - fn = create_internal ("octave_jit_--", scalar, scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *one = llvm::ConstantFP::get (scalar_t, 1); - llvm::Value *val = fn.argument (builder, 0); - val = builder.CreateFSub (val, one); - fn.do_return (builder, val); - } - unary_ops[octave_value::op_decr].add_overload (fn); - - fn = create_internal ("octave_jit_uminus", scalar, scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *mone = llvm::ConstantFP::get (scalar_t, -1); - llvm::Value *val = fn.argument (builder, 0); - val = builder.CreateFMul (val, mone); - fn.do_return (builder, val); - } - unary_ops[octave_value::op_uminus].add_overload (fn); - - fn = create_identity (scalar); - unary_ops[octave_value::op_uplus].add_overload (fn); - unary_ops[octave_value::op_transpose].add_overload (fn); - unary_ops[octave_value::op_hermitian].add_overload (fn); - - // now for binary complex operations - fn = create_internal ("octave_jit_+_complex_complex", complex, complex, - complex); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - llvm::Value *real = builder.CreateFAdd (complex_real (lhs), - complex_real (rhs)); - llvm::Value *imag = builder.CreateFAdd (complex_imag (lhs), - complex_imag (rhs)); - fn.do_return (builder, complex_new (real, imag)); - } - binary_ops[octave_value::op_add].add_overload (fn); - - fn = create_internal ("octave_jit_-_complex_complex", complex, complex, - complex); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - llvm::Value *real = builder.CreateFSub (complex_real (lhs), - complex_real (rhs)); - llvm::Value *imag = builder.CreateFSub (complex_imag (lhs), - complex_imag (rhs)); - fn.do_return (builder, complex_new (real, imag)); - } - binary_ops[octave_value::op_sub].add_overload (fn); - - fn = create_external (JIT_FN (octave_jit_complex_mul), - complex, complex, complex); - binary_ops[octave_value::op_mul].add_overload (fn); - binary_ops[octave_value::op_el_mul].add_overload (fn); - - jit_function complex_div = create_external (JIT_FN (octave_jit_complex_div), - complex, complex, complex); - complex_div.mark_can_error (); - binary_ops[octave_value::op_div].add_overload (fn); - binary_ops[octave_value::op_ldiv].add_overload (fn); - - fn = create_external (JIT_FN (octave_jit_pow_complex_complex), complex, - complex, complex); - binary_ops[octave_value::op_pow].add_overload (fn); - binary_ops[octave_value::op_el_pow].add_overload (fn); - - fn = create_internal ("octave_jit_*_scalar_complex", complex, scalar, - complex); - jit_function mul_scalar_complex = fn; - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::BasicBlock *complex_mul = fn.new_block ("complex_mul"); - llvm::BasicBlock *scalar_mul = fn.new_block ("scalar_mul"); - - llvm::Value *fzero = llvm::ConstantFP::get (scalar_t, 0); - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - - llvm::Value *cmp = builder.CreateFCmpUEQ (complex_imag (rhs), fzero); - builder.CreateCondBr (cmp, scalar_mul, complex_mul); - - builder.SetInsertPoint (scalar_mul); - llvm::Value *temp = complex_real (rhs); - temp = builder.CreateFMul (lhs, temp); - fn.do_return (builder, complex_new (temp, fzero), false); - - builder.SetInsertPoint (complex_mul); - temp = complex_new (builder.CreateFMul (lhs, complex_real (rhs)), - builder.CreateFMul (lhs, complex_imag (rhs))); - fn.do_return (builder, temp); - } - binary_ops[octave_value::op_mul].add_overload (fn); - binary_ops[octave_value::op_el_mul].add_overload (fn); - - fn = mirror_binary (mul_scalar_complex); - binary_ops[octave_value::op_mul].add_overload (fn); - binary_ops[octave_value::op_el_mul].add_overload (fn); - - fn = create_internal ("octave_jit_+_scalar_complex", complex, scalar, - complex); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - llvm::Value *real = builder.CreateFAdd (lhs, complex_real (rhs)); - fn.do_return (builder, complex_real (rhs, real)); - } - binary_ops[octave_value::op_add].add_overload (fn); - - fn = mirror_binary (fn); - binary_ops[octave_value::op_add].add_overload (fn); - - fn = create_internal ("octave_jit_-_complex_scalar", complex, complex, - scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - llvm::Value *real = builder.CreateFSub (complex_real (lhs), rhs); - fn.do_return (builder, complex_real (lhs, real)); - } - binary_ops[octave_value::op_sub].add_overload (fn); - - fn = create_internal ("octave_jit_-_scalar_complex", complex, scalar, - complex); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *lhs = fn.argument (builder, 0); - llvm::Value *rhs = fn.argument (builder, 1); - llvm::Value *real = builder.CreateFSub (lhs, complex_real (rhs)); - fn.do_return (builder, complex_real (rhs, real)); - } - binary_ops[octave_value::op_sub].add_overload (fn); - - fn = create_external (JIT_FN (octave_jit_pow_scalar_complex), complex, scalar, - complex); - binary_ops[octave_value::op_pow].add_overload (fn); - binary_ops[octave_value::op_el_pow].add_overload (fn); - - fn = create_external (JIT_FN (octave_jit_pow_complex_scalar), complex, - complex, scalar); - binary_ops[octave_value::op_pow].add_overload (fn); - binary_ops[octave_value::op_el_pow].add_overload (fn); - - // now for binary index operators - add_binary_op (index, octave_value::op_add, llvm::Instruction::Add); - - // and binary bool operators - add_binary_op (boolean, octave_value::op_el_or, llvm::Instruction::Or); - add_binary_op (boolean, octave_value::op_el_and, llvm::Instruction::And); - - // now for printing functions - print_fn.stash_name ("print"); - add_print (any, reinterpret_cast<void *> (&octave_jit_print_any)); - add_print (scalar, reinterpret_cast<void *> (&octave_jit_print_scalar)); - - // initialize for loop - for_init_fn.stash_name ("for_init"); - - fn = create_internal ("octave_jit_for_range_init", index, range); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *zero = llvm::ConstantInt::get (index_t, 0); - fn.do_return (builder, zero); - } - for_init_fn.add_overload (fn); - - // bounds check for for loop - for_check_fn.stash_name ("for_check"); - - fn = create_internal ("octave_jit_for_range_check", boolean, range, index); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *nelem - = builder.CreateExtractValue (fn.argument (builder, 0), 3); - llvm::Value *idx = fn.argument (builder, 1); - llvm::Value *ret = builder.CreateICmpULT (idx, nelem); - fn.do_return (builder, ret); - } - for_check_fn.add_overload (fn); - - // index variabe for for loop - for_index_fn.stash_name ("for_index"); - - fn = create_internal ("octave_jit_for_range_idx", scalar, range, index); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *idx = fn.argument (builder, 1); - llvm::Value *didx = builder.CreateSIToFP (idx, scalar_t); - llvm::Value *rng = fn.argument (builder, 0); - llvm::Value *base = builder.CreateExtractValue (rng, 0); - llvm::Value *inc = builder.CreateExtractValue (rng, 2); - - llvm::Value *ret = builder.CreateFMul (didx, inc); - ret = builder.CreateFAdd (base, ret); - fn.do_return (builder, ret); - } - for_index_fn.add_overload (fn); - - // logically true - logically_true_fn.stash_name ("logically_true"); - - jit_function gripe_nantl - = create_external (JIT_FN (octave_jit_octave::err_nan_to_logical_conversion), 0); - gripe_nantl.mark_can_error (); - - fn = create_internal ("octave_jit_logically_true_scalar", boolean, scalar); - fn.mark_can_error (); - - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::BasicBlock *error_block = fn.new_block ("error"); - llvm::BasicBlock *normal_block = fn.new_block ("normal"); - - llvm::Value *check = builder.CreateFCmpUNE (fn.argument (builder, 0), - fn.argument (builder, 0)); - builder.CreateCondBr (check, error_block, normal_block); - - builder.SetInsertPoint (error_block); - gripe_nantl.call (builder); - builder.CreateBr (normal_block); - builder.SetInsertPoint (normal_block); - - llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); - llvm::Value *ret = builder.CreateFCmpONE (fn.argument (builder, 0), zero); - fn.do_return (builder, ret); - } - logically_true_fn.add_overload (fn); - - // logically_true boolean - fn = create_identity (boolean); - logically_true_fn.add_overload (fn); - - // make_range - // FIXME: May be benificial to implement all in LLVM - make_range_fn.stash_name ("make_range"); - jit_function compute_nelem - = create_external (JIT_FN (octave_jit_compute_nelem), - index, scalar, scalar, scalar); - - fn = create_internal ("octave_jit_make_range", range, scalar, scalar, scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *base = fn.argument (builder, 0); - llvm::Value *limit = fn.argument (builder, 1); - llvm::Value *inc = fn.argument (builder, 2); - llvm::Value *nelem = compute_nelem.call (builder, base, limit, inc); - - llvm::Value *dzero = llvm::ConstantFP::get (scalar_t, 0); - llvm::Value *izero = llvm::ConstantInt::get (index_t, 0); - llvm::Value *rng = llvm::ConstantStruct::get (range_t, dzero, dzero, dzero, - izero, NULL); - rng = builder.CreateInsertValue (rng, base, 0); - rng = builder.CreateInsertValue (rng, limit, 1); - rng = builder.CreateInsertValue (rng, inc, 2); - rng = builder.CreateInsertValue (rng, nelem, 3); - fn.do_return (builder, rng); - } - make_range_fn.add_overload (fn); - - // paren_subsref - jit_type *jit_int = intN (sizeof (int) * 8); - llvm::Type *int_t = jit_int->to_llvm (); - jit_function ginvalid_index - = create_external (JIT_FN (octave_jit_ginvalid_index), 0); - jit_function gindex_range = create_external (JIT_FN (octave_jit_gindex_range), - 0, jit_int, jit_int, index, - index); - - fn = create_internal ("()subsref", scalar, matrix, scalar); - fn.mark_can_error (); - - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *one_idx = llvm::ConstantInt::get (index_t, 1); - llvm::Value *one_int = llvm::ConstantInt::get (int_t, 1); - - llvm::Value *undef = llvm::UndefValue::get (scalar_t); - llvm::Value *mat = fn.argument (builder, 0); - llvm::Value *idx = fn.argument (builder, 1); - - // convert index to scalar to integer, and check index >= 1 - llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); - llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); - llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); - llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one_idx); - llvm::Value *cond = builder.CreateOr (cond0, cond1); - - llvm::BasicBlock *done = fn.new_block ("done"); - llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); - llvm::BasicBlock *normal = fn.new_block ("normal", done); - builder.CreateCondBr (cond, conv_error, normal); - - builder.SetInsertPoint (conv_error); - ginvalid_index.call (builder); - builder.CreateBr (done); - - builder.SetInsertPoint (normal); - llvm::Value *len - = builder.CreateExtractValue (mat, llvm::ArrayRef<unsigned> (2)); - cond = builder.CreateICmpSGT (int_idx, len); - - llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); - llvm::BasicBlock *success = fn.new_block ("success", done); - builder.CreateCondBr (cond, bounds_error, success); - - builder.SetInsertPoint (bounds_error); - gindex_range.call (builder, one_int, one_int, int_idx, len); - builder.CreateBr (done); - - builder.SetInsertPoint (success); - llvm::Value *data = builder.CreateExtractValue (mat, - llvm::ArrayRef<unsigned> (1)); - llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); - llvm::Value *ret = builder.CreateLoad (gep); - builder.CreateBr (done); - - builder.SetInsertPoint (done); - - llvm::PHINode *merge = llvm::PHINode::Create (scalar_t, 3); - builder.Insert (merge); - merge->addIncoming (undef, conv_error); - merge->addIncoming (undef, bounds_error); - merge->addIncoming (ret, success); - fn.do_return (builder, merge); - } - paren_subsref_fn.add_overload (fn); - - // paren subsasgn - paren_subsasgn_fn.stash_name ("()subsasgn"); - - jit_function resize_paren_subsasgn - = create_external (JIT_FN (octave_jit_paren_subsasgn_impl), matrix, matrix, - index, scalar); - - fn = create_internal ("octave_jit_paren_subsasgn", matrix, matrix, scalar, - scalar); - fn.mark_can_error (); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *one_idx = llvm::ConstantInt::get (index_t, 1); - llvm::Value *one_int = llvm::ConstantInt::get (int_t, 1); - - llvm::Value *mat = fn.argument (builder, 0); - llvm::Value *idx = fn.argument (builder, 1); - llvm::Value *value = fn.argument (builder, 2); - - llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); - llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); - llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); - llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one_idx); - llvm::Value *cond = builder.CreateOr (cond0, cond1); - - llvm::BasicBlock *done = fn.new_block ("done"); - - llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); - llvm::BasicBlock *normal = fn.new_block ("normal", done); - builder.CreateCondBr (cond, conv_error, normal); - builder.SetInsertPoint (conv_error); - ginvalid_index.call (builder); - builder.CreateBr (done); - - builder.SetInsertPoint (normal); - llvm::Value *len = builder.CreateExtractValue (mat, 2); - cond0 = builder.CreateICmpSGT (int_idx, len); - - llvm::Value *rcount = builder.CreateExtractValue (mat, 0); - rcount = builder.CreateLoad (rcount); - cond1 = builder.CreateICmpSGT (rcount, one_int); - cond = builder.CreateOr (cond0, cond1); - - llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); - llvm::BasicBlock *success = fn.new_block ("success", done); - builder.CreateCondBr (cond, bounds_error, success); - - // resize on out of bounds access - builder.SetInsertPoint (bounds_error); - llvm::Value *resize_result = resize_paren_subsasgn.call (builder, mat, - int_idx, value); - builder.CreateBr (done); - - builder.SetInsertPoint (success); - llvm::Value *data - = builder.CreateExtractValue (mat, llvm::ArrayRef<unsigned> (1)); - llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); - builder.CreateStore (value, gep); - builder.CreateBr (done); - - builder.SetInsertPoint (done); - - llvm::PHINode *merge = llvm::PHINode::Create (matrix_t, 3); - builder.Insert (merge); - merge->addIncoming (mat, conv_error); - merge->addIncoming (resize_result, bounds_error); - merge->addIncoming (mat, success); - fn.do_return (builder, merge); - } - paren_subsasgn_fn.add_overload (fn); - - fn = create_external (JIT_FN (octave_jit_paren_subsasgn_matrix_range), matrix, - matrix, range, scalar); - fn.mark_can_error (); - paren_subsasgn_fn.add_overload (fn); - - end1_fn.stash_name ("end1"); - fn = create_internal ("octave_jit_end1_matrix", scalar, matrix, index, index); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *mat = fn.argument (builder, 0); - llvm::Value *ret = builder.CreateExtractValue (mat, 2); - fn.do_return (builder, builder.CreateSIToFP (ret, scalar_t)); - } - end1_fn.add_overload (fn); - - end_fn.stash_name ("end"); - fn = create_external (JIT_FN (octave_jit_end_matrix),scalar, matrix, index, - index); - end_fn.add_overload (fn); - - // -------------------- create_undef -------------------- - create_undef_fn.stash_name ("create_undef"); - fn = create_external (JIT_FN (octave_jit_create_undef), any); - create_undef_fn.add_overload (fn); - - casts[any->type_id ()].stash_name ("(any)"); - casts[scalar->type_id ()].stash_name ("(scalar)"); - casts[complex->type_id ()].stash_name ("(complex)"); - casts[matrix->type_id ()].stash_name ("(matrix)"); - casts[range->type_id ()].stash_name ("(range)"); - - // cast any <- matrix - fn = create_external (JIT_FN (octave_jit_cast_any_matrix), any, matrix); - casts[any->type_id ()].add_overload (fn); - - // cast matrix <- any - fn = create_external (JIT_FN (octave_jit_cast_matrix_any), matrix, any); - casts[matrix->type_id ()].add_overload (fn); - - // cast any <- range - fn = create_external (JIT_FN (octave_jit_cast_any_range), any, range); - casts[any->type_id ()].add_overload (fn); - - // cast range <- any - fn = create_external (JIT_FN (octave_jit_cast_range_any), range, any); - casts[range->type_id ()].add_overload (fn); - - // cast any <- scalar - fn = create_external (JIT_FN (octave_jit_cast_any_scalar), any, scalar); - casts[any->type_id ()].add_overload (fn); - - // cast scalar <- any - fn = create_external (JIT_FN (octave_jit_cast_scalar_any), scalar, any); - casts[scalar->type_id ()].add_overload (fn); - - // cast any <- complex - fn = create_external (JIT_FN (octave_jit_cast_any_complex), any, complex); - casts[any->type_id ()].add_overload (fn); - - // cast complex <- any - fn = create_external (JIT_FN (octave_jit_cast_complex_any), complex, any); - casts[complex->type_id ()].add_overload (fn); - - // cast complex <- scalar - fn = create_internal ("octave_jit_cast_complex_scalar", complex, scalar); - body = fn.new_block (); - builder.SetInsertPoint (body); - { - llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); - fn.do_return (builder, complex_new (fn.argument (builder, 0), zero)); - } - casts[complex->type_id ()].add_overload (fn); - - // cast scalar <- complex - fn = create_internal ("octave_jit_cast_scalar_complex", scalar, complex); - body = fn.new_block (); - builder.SetInsertPoint (body); - fn.do_return (builder, complex_real (fn.argument (builder, 0))); - casts[scalar->type_id ()].add_overload (fn); - - // cast any <- any - fn = create_identity (any); - casts[any->type_id ()].add_overload (fn); - - // cast scalar <- scalar - fn = create_identity (scalar); - casts[scalar->type_id ()].add_overload (fn); - - // cast complex <- complex - fn = create_identity (complex); - casts[complex->type_id ()].add_overload (fn); - - // -------------------- builtin functions -------------------- - add_builtin ("#unknown_function"); - unknown_function = builtins["#unknown_function"]; - - add_builtin ("sin"); - register_intrinsic ("sin", llvm::Intrinsic::sin, scalar, scalar); - register_generic ("sin", matrix, matrix); - - add_builtin ("cos"); - register_intrinsic ("cos", llvm::Intrinsic::cos, scalar, scalar); - register_generic ("cos", matrix, matrix); - - add_builtin ("exp"); - register_intrinsic ("exp", llvm::Intrinsic::exp, scalar, scalar); - register_generic ("exp", matrix, matrix); - - add_builtin ("balance"); - register_generic ("balance", matrix, matrix); - - add_builtin ("cond"); - register_generic ("cond", scalar, matrix); - - add_builtin ("det"); - register_generic ("det", scalar, matrix); - - add_builtin ("norm"); - register_generic ("norm", scalar, matrix); - - add_builtin ("rand"); - register_generic ("rand", matrix, scalar); - register_generic ("rand", matrix, std::vector<jit_type *> (2, scalar)); - - add_builtin ("magic"); - register_generic ("magic", matrix, scalar); - register_generic ("magic", matrix, std::vector<jit_type *> (2, scalar)); - - add_builtin ("eye"); - register_generic ("eye", matrix, scalar); - register_generic ("eye", matrix, std::vector<jit_type *> (2, scalar)); - - add_builtin ("mod"); - register_generic ("mod", scalar, std::vector<jit_type *> (2, scalar)); - - casts.resize (next_id + 1); - jit_function any_id = create_identity (any); - jit_function grab_any = create_external (JIT_FN (octave_jit_grab_any), - any, any); - jit_function release_any = get_release (any); - std::vector<jit_type *> args; - args.resize (1); - - for (std::map<std::string, jit_type *>::iterator iter = builtins.begin (); - iter != builtins.end (); ++iter) - { - jit_type *btype = iter->second; - args[0] = btype; - - grab_fn.add_overload (jit_function (grab_any, btype, args)); - release_fn.add_overload (jit_function (release_any, 0, args)); - casts[any->type_id ()].add_overload (jit_function (any_id, any, args)); - - args[0] = any; - casts[btype->type_id ()].add_overload (jit_function (any_id, btype, - args)); - } -} - -const jit_function& -jit_typeinfo::do_end (jit_value *value, jit_value *idx, jit_value *count) -{ - jit_const_index *ccount = dynamic_cast<jit_const_index *> (count); - if (ccount && ccount->value () == 1) - return end1_fn.overload (value->type (), idx->type (), count->type ()); - - return end_fn.overload (value->type (), idx->type (), count->type ()); -} - -jit_type* -jit_typeinfo::new_type (const std::string& name, jit_type *parent, - llvm::Type *llvm_type, bool skip_paren) -{ - jit_type *ret = new jit_type (name, parent, llvm_type, skip_paren, next_id++); - id_to_type.push_back (ret); - return ret; -} - -void -jit_typeinfo::add_print (jit_type *ty, void *fptr) -{ - std::stringstream name; - name << "octave_jit_print_" << ty->name (); - jit_function fn = create_external (engine, fptr, name.str (), - 0, intN (8), ty); - print_fn.add_overload (fn); -} - -// FIXME: cp between add_binary_op, add_binary_icmp, and add_binary_fcmp -void -jit_typeinfo::add_binary_op (jit_type *ty, int op, int llvm_op) -{ - std::stringstream fname; - octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); - fname << "octave_jit_" << octave_value::binary_op_as_string (ov_op) - << "_" << ty->name (); - - jit_function fn = create_internal (fname.str (), ty, ty, ty); - llvm::BasicBlock *block = fn.new_block (); - builder.SetInsertPoint (block); - llvm::Instruction::BinaryOps temp - = static_cast<llvm::Instruction::BinaryOps>(llvm_op); - - llvm::Value *ret = builder.CreateBinOp (temp, fn.argument (builder, 0), - fn.argument (builder, 1)); - fn.do_return (builder, ret); - binary_ops[op].add_overload (fn); -} - -void -jit_typeinfo::add_binary_icmp (jit_type *ty, int op, int llvm_op) -{ - std::stringstream fname; - octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); - fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) - << "_" << ty->name (); - - jit_function fn = create_internal (fname.str (), boolean, ty, ty); - llvm::BasicBlock *block = fn.new_block (); - builder.SetInsertPoint (block); - llvm::CmpInst::Predicate temp - = static_cast<llvm::CmpInst::Predicate>(llvm_op); - llvm::Value *ret = builder.CreateICmp (temp, fn.argument (builder, 0), - fn.argument (builder, 1)); - fn.do_return (builder, ret); - binary_ops[op].add_overload (fn); -} - -void -jit_typeinfo::add_binary_fcmp (jit_type *ty, int op, int llvm_op) -{ - std::stringstream fname; - octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); - fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) - << "_" << ty->name (); - - jit_function fn = create_internal (fname.str (), boolean, ty, ty); - llvm::BasicBlock *block = fn.new_block (); - builder.SetInsertPoint (block); - llvm::CmpInst::Predicate temp - = static_cast<llvm::CmpInst::Predicate>(llvm_op); - llvm::Value *ret = builder.CreateFCmp (temp, fn.argument (builder, 0), - fn.argument (builder, 1)); - fn.do_return (builder, ret); - binary_ops[op].add_overload (fn); -} - -jit_function -jit_typeinfo::create_function (jit_convention::type cc, const llvm::Twine& name, - jit_type *ret, - const std::vector<jit_type *>& args) -{ - jit_function result (module, cc, name, ret, args); - return result; -} - -jit_function -jit_typeinfo::create_identity (jit_type *type) -{ - size_t id = type->type_id (); - if (id >= identities.size ()) - identities.resize (id + 1); - - if (! identities[id].valid ()) - { - std::stringstream name; - name << "id_" << type->name (); - - jit_function fn = create_internal (name.str (), type, type); - llvm::BasicBlock *body = fn.new_block (); - builder.SetInsertPoint (body); - fn.do_return (builder, fn.argument (builder, 0)); - return identities[id] = fn; - } - - return identities[id]; -} - -llvm::Value * -jit_typeinfo::do_insert_error_check (llvm::IRBuilderD& abuilder) -{ - return abuilder.CreateLoad (lerror_state); -} - -llvm::Value * -jit_typeinfo::do_insert_interrupt_check (llvm::IRBuilderD& abuilder) -{ - llvm::LoadInst *val = abuilder.CreateLoad (loctave_interrupt_state); - val->setVolatile (true); - return abuilder.CreateICmpSGT (val, abuilder.getInt32 (0)); -} - -void -jit_typeinfo::add_builtin (const std::string& name) -{ - jit_type *btype = new_type (name, any, any->to_llvm (), true); - builtins[name] = btype; - - octave_builtin *ov_builtin = find_builtin (name); - if (ov_builtin) - ov_builtin->stash_jit (*btype); -} - -void -jit_typeinfo::register_intrinsic (const std::string& name, size_t iid, - jit_type *result, - const std::vector<jit_type *>& args) -{ - jit_type *builtin_type = builtins[name]; - size_t nargs = args.size (); - llvm::SmallVector<llvm::Type *, 5> llvm_args (nargs); - for (size_t i = 0; i < nargs; ++i) - llvm_args[i] = args[i]->to_llvm (); - - llvm::Intrinsic::ID id = static_cast<llvm::Intrinsic::ID> (iid); - llvm::Function *ifun = llvm::Intrinsic::getDeclaration (module, id, - llvm_args); - std::stringstream fn_name; - fn_name << "octave_jit_" << name; - - std::vector<jit_type *> args1 (nargs + 1); - args1[0] = builtin_type; - std::copy (args.begin (), args.end (), args1.begin () + 1); - - // The first argument will be the Octave function, but we already know that - // the function call is the equivalent of the intrinsic, so we ignore it and - // call the intrinsic with the remaining arguments. - jit_function fn = create_internal (fn_name.str (), result, args1); - llvm::BasicBlock *body = fn.new_block (); - builder.SetInsertPoint (body); - - llvm::SmallVector<llvm::Value *, 5> fargs (nargs); - for (size_t i = 0; i < nargs; ++i) - fargs[i] = fn.argument (builder, i + 1); - - llvm::Value *ret = builder.CreateCall (ifun, fargs); - fn.do_return (builder, ret); - paren_subsref_fn.add_overload (fn); -} - -octave_builtin * -jit_typeinfo::find_builtin (const std::string& name) -{ - // FIXME: Finalize what we want to store in octave_builtin, then add functions - // to access these values in octave_value - octave_value ov_builtin = symbol_table::find (name); - return dynamic_cast<octave_builtin *> (ov_builtin.internal_rep ()); -} - -void -jit_typeinfo::register_generic (const std::string& name, jit_type *result, - const std::vector<jit_type *>& args) -{ - octave_builtin *builtin = find_builtin (name); - if (! builtin) - return; - - std::vector<jit_type *> fn_args (args.size () + 1); - fn_args[0] = builtins[name]; - std::copy (args.begin (), args.end (), fn_args.begin () + 1); - jit_function fn = create_internal (name, result, fn_args); - fn.mark_can_error (); - llvm::BasicBlock *block = fn.new_block (); - builder.SetInsertPoint (block); - llvm::Type *any_t = any->to_llvm (); - llvm::ArrayType *array_t = llvm::ArrayType::get (any_t, args.size ()); - llvm::Value *array = llvm::UndefValue::get (array_t); - for (size_t i = 0; i < args.size (); ++i) - { - llvm::Value *arg = fn.argument (builder, i + 1); - jit_function agrab = get_grab (args[i]); - if (agrab.valid ()) - arg = agrab.call (builder, arg); - jit_function acast = cast (any, args[i]); - array = builder.CreateInsertValue (array, acast.call (builder, arg), i); - } - - llvm::Value *array_mem = builder.CreateAlloca (array_t); - builder.CreateStore (array, array_mem); - array = builder.CreateBitCast (array_mem, any_t->getPointerTo ()); - - jit_type *jintTy = intN (sizeof (octave_builtin::fcn) * 8); - llvm::Type *intTy = jintTy->to_llvm (); - size_t fcn_int = reinterpret_cast<size_t> (builtin->function ()); - llvm::Value *fcn = llvm::ConstantInt::get (intTy, fcn_int); - llvm::Value *nargin = llvm::ConstantInt::get (intTy, args.size ()); - size_t result_int = reinterpret_cast<size_t> (result); - llvm::Value *res_llvm = llvm::ConstantInt::get (intTy, result_int); - llvm::Value *ret = any_call.call (builder, fcn, nargin, array, res_llvm); - - jit_function cast_result = cast (result, any); - fn.do_return (builder, cast_result.call (builder, ret)); - paren_subsref_fn.add_overload (fn); -} - -jit_function -jit_typeinfo::mirror_binary (const jit_function& fn) -{ - jit_function ret = create_internal (fn.name () + "_reverse", - fn.result (), fn.argument_type (1), - fn.argument_type (0)); - if (fn.can_error ()) - ret.mark_can_error (); - - llvm::BasicBlock *body = ret.new_block (); - builder.SetInsertPoint (body); - llvm::Value *result = fn.call (builder, ret.argument (builder, 1), - ret.argument (builder, 0)); - if (ret.result ()) - ret.do_return (builder, result); - else - ret.do_return (builder); - - return ret; -} - -llvm::Value * -jit_typeinfo::pack_complex (llvm::IRBuilderD& bld, llvm::Value *cplx) -{ - llvm::Type *complex_ret = instance->complex_ret; - llvm::Value *real = bld.CreateExtractValue (cplx, 0); - llvm::Value *imag = bld.CreateExtractValue (cplx, 1); - llvm::Value *ret = llvm::UndefValue::get (complex_ret); - - unsigned int re_idx[] = {0, 0}; - unsigned int im_idx[] = {0, 1}; - ret = bld.CreateInsertValue (ret, real, re_idx); - return bld.CreateInsertValue (ret, imag, im_idx); -} - -llvm::Value * -jit_typeinfo::unpack_complex (llvm::IRBuilderD& bld, llvm::Value *result) -{ - unsigned int re_idx[] = {0, 0}; - unsigned int im_idx[] = {0, 1}; - - llvm::Type *complex_t = get_complex ()->to_llvm (); - llvm::Value *real = bld.CreateExtractValue (result, re_idx); - llvm::Value *imag = bld.CreateExtractValue (result, im_idx); - llvm::Value *ret = llvm::UndefValue::get (complex_t); - - ret = bld.CreateInsertValue (ret, real, 0); - return bld.CreateInsertValue (ret, imag, 1); -} - -llvm::Value * -jit_typeinfo::complex_real (llvm::Value *cx) -{ - return builder.CreateExtractValue (cx, 0); -} - -llvm::Value * -jit_typeinfo::complex_real (llvm::Value *cx, llvm::Value *real) -{ - return builder.CreateInsertValue (cx, real, 0); -} - -llvm::Value * -jit_typeinfo::complex_imag (llvm::Value *cx) -{ - return builder.CreateExtractValue (cx, 1); -} - -llvm::Value * -jit_typeinfo::complex_imag (llvm::Value *cx, llvm::Value *imag) -{ - return builder.CreateInsertValue (cx, imag, 1); -} - -llvm::Value * -jit_typeinfo::complex_new (llvm::Value *real, llvm::Value *imag) -{ - llvm::Value *ret = llvm::UndefValue::get (complex->to_llvm ()); - ret = complex_real (ret, real); - return complex_imag (ret, imag); -} - -void -jit_typeinfo::create_int (size_t nbits) -{ - std::stringstream tname; - tname << "int" << nbits; - ints[nbits] = new_type (tname.str (), any, llvm::Type::getIntNTy (context, - nbits)); -} - -jit_type * -jit_typeinfo::intN (size_t nbits) const -{ - std::map<size_t, jit_type *>::const_iterator iter = ints.find (nbits); - if (iter != ints.end ()) - return iter->second; - - throw jit_fail_exception ("No such integer type"); -} - -jit_type * -jit_typeinfo::do_type_of (const octave_value &ov) const -{ - if (ov.is_function ()) - { - // FIXME: This is ugly, we need to finalize how we want to do this, then - // have octave_value fully support the needed functionality - octave_builtin *builtin - = dynamic_cast<octave_builtin *> (ov.internal_rep ()); - return builtin && builtin->to_jit () ? builtin->to_jit () - : unknown_function; - } - - if (ov.is_range ()) - return get_range (); - - if (ov.is_double_type () && ! ov.is_complex_type ()) - { - if (ov.is_real_scalar ()) - return get_scalar (); - - if (ov.is_matrix_type ()) - return get_matrix (); - } - - if (ov.is_complex_scalar ()) - { - Complex cv = ov.complex_value (); - - // We don't really represent complex values, instead we represent - // complex_or_scalar. If the imag value is zero, we assume a scalar. - if (cv.imag () != 0) - return get_complex (); - } - - return get_any (); -} - -#endif
--- a/libinterp/corefcn/jit-typeinfo.h Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,851 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -#if ! defined (octave_jit_typeinfo_h) -#define octave_jit_typeinfo_h 1 - -#include "octave-config.h" - -#if defined (HAVE_LLVM) - -#include <map> -#include <vector> - -#include "Range.h" -#include "jit-util.h" - -// Defines the type system used by jit and a singleton class, jit_typeinfo, to -// manage the types. -// -// FIXME: -// Operations are defined and implemented in jit_typeinfo. Eventually they -// should be moved elsewhere. (just like with octave_typeinfo) - -// jit_range is compatible with the llvm range structure -struct -jit_range -{ - jit_range (const Range& from) : base (from.base ()), limit (from.limit ()), - inc (from.inc ()), nelem (from.numel ()) - { } - - operator Range () const - { - return Range (base, limit, inc); - } - - bool all_elements_are_ints () const; - - double base; - double limit; - double inc; - octave_idx_type nelem; -}; - -std::ostream& operator << (std::ostream& os, const jit_range& rng); - -// jit_array is compatible with the llvm array/matrix structures -template <typename T, typename U> -struct -jit_array -{ - jit_array () : array (0) { } - - jit_array (T& from) : array (new T (from)) - { - update (); - } - - void update (void) - { - ref_count = array->jit_ref_count (); - slice_data = array->jit_slice_data () - 1; - slice_len = array->numel (); - dimensions = array->jit_dimensions (); - } - - void update (T *aarray) - { - array = aarray; - update (); - } - - operator T () const - { - return *array; - } - - int *ref_count; - - U *slice_data; - octave_idx_type slice_len; - octave_idx_type *dimensions; - - T *array; -}; - -typedef jit_array<NDArray, double> jit_matrix; - -std::ostream& operator << (std::ostream& os, const jit_matrix& mat); - -// calling convention -namespace jit_convention -{ - enum - type - { - // internal to jit - internal, - - // an external C call - external, - - length - }; -} - -// Used to keep track of estimated (infered) types during JIT. This is a -// hierarchical type system which includes both concrete and abstract types. -// -// The types form a lattice. Currently we only allow for one parent type, but -// eventually we may allow for multiple predecessors. -class -jit_type -{ -public: - typedef llvm::Value *(*convert_fn) (llvm::IRBuilderD&, llvm::Value *); - - jit_type (const std::string& aname, jit_type *aparent, llvm::Type *allvm_type, - bool askip_paren, int aid); - - // a user readable type name - const std::string& name (void) const { return mname; } - - // a unique id for the type - int type_id (void) const { return mid; } - - // An abstract base type, may be null - jit_type *parent (void) const { return mparent; } - - // convert to an llvm type - llvm::Type *to_llvm (void) const { return llvm_type; } - - // how this type gets passed as a function argument - llvm::Type *to_llvm_arg (void) const; - - size_t depth (void) const { return mdepth; } - - bool skip_paren (void) const { return mskip_paren; } - - // -------------------- Calling Convention information -------------------- - - // A function declared like: mytype foo (int arg0, int arg1); - // Will be converted to: void foo (mytype *retval, int arg0, int arg1) - // if mytype is sret. The caller is responsible for allocating space for - // retval. (on the stack) - bool sret (jit_convention::type cc) const { return msret[cc]; } - - void mark_sret (jit_convention::type cc) - { msret[cc] = true; } - - // A function like: void foo (mytype arg0) - // Will be converted to: void foo (mytype *arg0) - // Basically just pass by reference. - bool pointer_arg (jit_convention::type cc) const { return mpointer_arg[cc]; } - - void mark_pointer_arg (jit_convention::type cc) - { mpointer_arg[cc] = true; } - - // Convert into an equivalent form before calling. For example, complex is - // represented as two values llvm vector, but we need to pass it as a two - // valued llvm structure to C functions. - convert_fn pack (jit_convention::type cc) { return mpack[cc]; } - - void set_pack (jit_convention::type cc, convert_fn fn) { mpack[cc] = fn; } - - // The inverse operation of pack. - convert_fn unpack (jit_convention::type cc) { return munpack[cc]; } - - void set_unpack (jit_convention::type cc, convert_fn fn) - { munpack[cc] = fn; } - - // The resulting type after pack is called. - llvm::Type *packed_type (jit_convention::type cc) - { return mpacked_type[cc]; } - - void set_packed_type (jit_convention::type cc, llvm::Type *ty) - { mpacked_type[cc] = ty; } -private: - std::string mname; - jit_type *mparent; - llvm::Type *llvm_type; - int mid; - size_t mdepth; - bool mskip_paren; - - bool msret[jit_convention::length]; - bool mpointer_arg[jit_convention::length]; - - convert_fn mpack[jit_convention::length]; - convert_fn munpack[jit_convention::length]; - - llvm::Type *mpacked_type[jit_convention::length]; -}; - -// seperate print function to allow easy printing if type is null -std::ostream& jit_print (std::ostream& os, jit_type *atype); - -class jit_value; - -// An abstraction for calling llvm functions with jit_values. Deals with -// calling convention details. -class -jit_function -{ - friend std::ostream& operator << (std::ostream& os, const jit_function& fn); -public: - // create a function in an invalid state - jit_function (); - - jit_function (llvm::Module *amodule, jit_convention::type acall_conv, - const llvm::Twine& aname, jit_type *aresult, - const std::vector<jit_type *>& aargs); - - // Use an existing function, but change the argument types. The new argument - // types must behave the same for the current calling convention. - jit_function (const jit_function& fn, jit_type *aresult, - const std::vector<jit_type *>& aargs); - - jit_function (const jit_function& fn); - - // erase the interal LLVM function (if it exists). Will become invalid. - void erase (void); - - template <typename T> - void add_mapping (llvm::ExecutionEngine *engine, T fn) - { - do_add_mapping (engine, reinterpret_cast<void *> (fn)); - } - - bool valid (void) const { return llvm_function; } - - std::string name (void) const; - - llvm::BasicBlock *new_block (const std::string& aname = "body", - llvm::BasicBlock *insert_before = 0); - - llvm::Value *call (llvm::IRBuilderD& builder, - const std::vector<jit_value *>& in_args) const; - - llvm::Value *call (llvm::IRBuilderD& builder, - const std::vector<llvm::Value *>& in_args - = std::vector<llvm::Value *> ()) const; - -#define JIT_PARAM_ARGS llvm::IRBuilderD& builder, -#define JIT_PARAMS builder, -#define JIT_CALL(N) JIT_EXPAND (llvm::Value *, call, llvm::Value *, const, N) - - JIT_CALL (1) - JIT_CALL (2) - JIT_CALL (3) - JIT_CALL (4) - JIT_CALL (5) - -#undef JIT_CALL - -#define JIT_CALL(N) JIT_EXPAND (llvm::Value *, call, jit_value *, const, N) - - JIT_CALL (1); - JIT_CALL (2); - JIT_CALL (3); - -#undef JIT_CALL -#undef JIT_PARAMS -#undef JIT_PARAM_ARGS - - llvm::Value *argument (llvm::IRBuilderD& builder, size_t idx) const; - - void do_return (llvm::IRBuilderD& builder, llvm::Value *rval = 0, - bool verify = true); - - llvm::Function *to_llvm (void) const { return llvm_function; } - - // If true, then the return value is passed as a pointer in the first argument - bool sret (void) const { return mresult && mresult->sret (call_conv); } - - bool can_error (void) const { return mcan_error; } - - void mark_can_error (void) { mcan_error = true; } - - jit_type *result (void) const { return mresult; } - - jit_type *argument_type (size_t idx) const - { - assert (idx < args.size ()); - return args[idx]; - } - - const std::vector<jit_type *>& arguments (void) const { return args; } -private: - void do_add_mapping (llvm::ExecutionEngine *engine, void *fn); - - llvm::Module *module; - llvm::Function *llvm_function; - jit_type *mresult; - std::vector<jit_type *> args; - jit_convention::type call_conv; - bool mcan_error; -}; - -std::ostream& operator << (std::ostream& os, const jit_function& fn); - -// Keeps track of information about how to implement operations (+, -, *, ect) -// and their resulting types. -class -jit_operation -{ -public: - // type signature vector - typedef std::vector<jit_type *> signature_vec; - - virtual ~jit_operation (void); - - void add_overload (const jit_function& func) - { - add_overload (func, func.arguments ()); - } - - void add_overload (const jit_function& func, - const signature_vec& args); - - const jit_function& overload (const signature_vec& types) const; - - jit_type *result (const signature_vec& types) const - { - const jit_function& temp = overload (types); - return temp.result (); - } - -#define JIT_PARAMS -#define JIT_PARAM_ARGS -#define JIT_OVERLOAD(N) \ - JIT_EXPAND (const jit_function&, overload, jit_type *, const, N) \ - JIT_EXPAND (jit_type *, result, jit_type *, const, N) - - JIT_OVERLOAD (1); - JIT_OVERLOAD (2); - JIT_OVERLOAD (3); - -#undef JIT_PARAMS -#undef JIT_PARAM_ARGS - - const std::string& name (void) const { return mname; } - - void stash_name (const std::string& aname) { mname = aname; } -protected: - virtual jit_function *generate (const signature_vec& types) const; -private: - Array<octave_idx_type> to_idx (const signature_vec& types) const; - - const jit_function& do_generate (const signature_vec& types) const; - - struct signature_cmp - { - bool operator () (const signature_vec *lhs, const signature_vec *rhs) const; - }; - - typedef std::map<const signature_vec *, jit_function *, signature_cmp> - generated_map; - - mutable generated_map generated; - - std::vector<Array<jit_function> > overloads; - - std::string mname; -}; - -class -jit_index_operation : public jit_operation -{ -public: - jit_index_operation (void) : module (0), engine (0) { } - - void initialize (llvm::Module *amodule, llvm::ExecutionEngine *aengine) - { - module = amodule; - engine = aengine; - do_initialize (); - } -protected: - virtual jit_function *generate (const signature_vec& types) const; - - virtual jit_function *generate_matrix (const signature_vec& types) const = 0; - - virtual void do_initialize (void) = 0; - - // helper functions - // [start_idx, end_idx). - llvm::Value *create_arg_array (llvm::IRBuilderD& builder, - const jit_function &fn, size_t start_idx, - size_t end_idx) const; - - llvm::Module *module; - llvm::ExecutionEngine *engine; -}; - -class -jit_paren_subsref : public jit_index_operation -{ -protected: - virtual jit_function *generate_matrix (const signature_vec& types) const; - - virtual void do_initialize (void); -private: - jit_function paren_scalar; -}; - -class -jit_paren_subsasgn : public jit_index_operation -{ -protected: - jit_function *generate_matrix (const signature_vec& types) const; - - virtual void do_initialize (void); -private: - jit_function paren_scalar; -}; - -// A singleton class which handles the construction of jit_types and -// jit_operations. -class -jit_typeinfo -{ -public: - static void initialize (llvm::Module *m, llvm::ExecutionEngine *e); - - static jit_type *join (jit_type *lhs, jit_type *rhs) - { - return instance->do_join (lhs, rhs); - } - - static jit_type *get_any (void) { return instance->any; } - - static jit_type *get_matrix (void) { return instance->matrix; } - - static jit_type *get_scalar (void) { return instance->scalar; } - - static llvm::Type *get_scalar_llvm (void) - { return instance->scalar->to_llvm (); } - - static jit_type *get_scalar_ptr (void) { return instance->scalar_ptr; } - - static jit_type *get_any_ptr (void) { return instance->any_ptr; } - - static jit_type *get_range (void) { return instance->range; } - - static jit_type *get_string (void) { return instance->string; } - - static jit_type *get_bool (void) { return instance->boolean; } - - static jit_type *get_index (void) { return instance->index; } - - static llvm::Type *get_index_llvm (void) - { return instance->index->to_llvm (); } - - static jit_type *get_complex (void) { return instance->complex; } - - // Get the jit_type of an octave_value - static jit_type *type_of (const octave_value& ov) - { - return instance->do_type_of (ov); - } - - static const jit_operation& binary_op (int op) - { - return instance->do_binary_op (op); - } - - static const jit_operation& unary_op (int op) - { - return instance->do_unary_op (op); - } - - static const jit_operation& grab (void) { return instance->grab_fn; } - - static const jit_function& get_grab (jit_type *type) - { - return instance->grab_fn.overload (type); - } - - static const jit_operation& release (void) - { - return instance->release_fn; - } - - static const jit_function& get_release (jit_type *type) - { - return instance->release_fn.overload (type); - } - - static const jit_operation& destroy (void) - { - return instance->destroy_fn; - } - - static const jit_operation& print_value (void) - { - return instance->print_fn; - } - - static const jit_operation& for_init (void) - { - return instance->for_init_fn; - } - - static const jit_operation& for_check (void) - { - return instance->for_check_fn; - } - - static const jit_operation& for_index (void) - { - return instance->for_index_fn; - } - - static const jit_operation& make_range (void) - { - return instance->make_range_fn; - } - - static const jit_operation& paren_subsref (void) - { - return instance->paren_subsref_fn; - } - - static const jit_operation& paren_subsasgn (void) - { - return instance->paren_subsasgn_fn; - } - - static const jit_operation& logically_true (void) - { - return instance->logically_true_fn; - } - - static const jit_operation& cast (jit_type *result) - { - return instance->do_cast (result); - } - - static const jit_function& cast (jit_type *to, jit_type *from) - { - return instance->do_cast (to, from); - } - - static llvm::Value *insert_error_check (llvm::IRBuilderD& bld) - { - return instance->do_insert_error_check (bld); - } - - static llvm::Value *insert_interrupt_check (llvm::IRBuilderD& bld) - { - return instance->do_insert_interrupt_check (bld); - } - - static const jit_operation& end (void) - { - return instance->end_fn; - } - - static const jit_function& end (jit_value *value, jit_value *index, - jit_value *count) - { - return instance->do_end (value, index, count); - } - - static const jit_operation& create_undef (void) - { - return instance->create_undef_fn; - } - - static llvm::Value *create_complex (llvm::Value *real, llvm::Value *imag) - { - return instance->complex_new (real, imag); - } -private: - jit_typeinfo (llvm::Module *m, llvm::ExecutionEngine *e); - - // FIXME: Do these methods really need to be in jit_typeinfo? - jit_type *do_join (jit_type *lhs, jit_type *rhs) - { - // empty case - if (! lhs) - return rhs; - - if (! rhs) - return lhs; - - // check for a shared parent - while (lhs != rhs) - { - if (lhs->depth () > rhs->depth ()) - lhs = lhs->parent (); - else if (lhs->depth () < rhs->depth ()) - rhs = rhs->parent (); - else - { - // we MUST have depth > 0 as any is the base type of everything - do - { - lhs = lhs->parent (); - rhs = rhs->parent (); - } - while (lhs != rhs); - } - } - - return lhs; - } - - jit_type *do_difference (jit_type *lhs, jit_type *) - { - // FIXME: Maybe we can do something smarter? - return lhs; - } - - jit_type *do_type_of (const octave_value &ov) const; - - const jit_operation& do_binary_op (int op) const - { - assert (static_cast<size_t>(op) < binary_ops.size ()); - return binary_ops[op]; - } - - const jit_operation& do_unary_op (int op) const - { - assert (static_cast<size_t> (op) < unary_ops.size ()); - return unary_ops[op]; - } - - const jit_operation& do_cast (jit_type *to) - { - static jit_operation null_function; - if (! to) - return null_function; - - size_t id = to->type_id (); - if (id >= casts.size ()) - return null_function; - return casts[id]; - } - - const jit_function& do_cast (jit_type *to, jit_type *from) - { - return do_cast (to).overload (from); - } - - const jit_function& do_end (jit_value *value, jit_value *index, - jit_value *count); - - jit_type *new_type (const std::string& name, jit_type *parent, - llvm::Type *llvm_type, bool skip_paren = false); - - void add_print (jit_type *ty, void *fptr); - - void add_binary_op (jit_type *ty, int op, int llvm_op); - - void add_binary_icmp (jit_type *ty, int op, int llvm_op); - - void add_binary_fcmp (jit_type *ty, int op, int llvm_op); - - // create a function with an external calling convention - // forces the function pointer to be specified - template <typename T> - jit_function create_external (llvm::ExecutionEngine *ee, T fn, - const llvm::Twine& name, jit_type *ret, - const std::vector<jit_type *>& args - = std::vector<jit_type *> ()) - { - jit_function retval = create_function (jit_convention::external, name, ret, - args); - retval.add_mapping (ee, fn); - return retval; - } - -#define JIT_PARAM_ARGS llvm::ExecutionEngine *ee, T fn, \ - const llvm::Twine& name, jit_type *ret, -#define JIT_PARAMS ee, fn, name, ret, -#define CREATE_FUNCTION(N) JIT_EXPAND(template <typename T> jit_function, \ - create_external, \ - jit_type *, /* empty */, N) - - CREATE_FUNCTION(1); - CREATE_FUNCTION(2); - CREATE_FUNCTION(3); - CREATE_FUNCTION(4); - -#undef JIT_PARAM_ARGS -#undef JIT_PARAMS -#undef CREATE_FUNCTION - - // use create_external or create_internal directly - jit_function create_function (jit_convention::type cc, - const llvm::Twine& name, jit_type *ret, - const std::vector<jit_type *>& args - = std::vector<jit_type *> ()); - - // create an internal calling convention (a function defined in llvm) - jit_function create_internal (const llvm::Twine& name, jit_type *ret, - const std::vector<jit_type *>& args - = std::vector<jit_type *> ()) - { - return create_function (jit_convention::internal, name, ret, args); - } - -#define JIT_PARAM_ARGS const llvm::Twine& name, jit_type *ret, -#define JIT_PARAMS name, ret, -#define CREATE_FUNCTION(N) JIT_EXPAND(jit_function, create_internal, \ - jit_type *, /* empty */, N) - - CREATE_FUNCTION(1); - CREATE_FUNCTION(2); - CREATE_FUNCTION(3); - CREATE_FUNCTION(4); - -#undef JIT_PARAM_ARGS -#undef JIT_PARAMS -#undef CREATE_FUNCTION - - jit_function create_identity (jit_type *type); - - llvm::Value *do_insert_error_check (llvm::IRBuilderD& bld); - - llvm::Value *do_insert_interrupt_check (llvm::IRBuilderD& bld); - - void add_builtin (const std::string& name); - - void register_intrinsic (const std::string& name, size_t id, - jit_type *result, jit_type *arg0) - { - std::vector<jit_type *> args (1, arg0); - register_intrinsic (name, id, result, args); - } - - void register_intrinsic (const std::string& name, size_t id, jit_type *result, - const std::vector<jit_type *>& args); - - void register_generic (const std::string& name, jit_type *result, - jit_type *arg0) - { - std::vector<jit_type *> args (1, arg0); - register_generic (name, result, args); - } - - void register_generic (const std::string& name, jit_type *result, - const std::vector<jit_type *>& args); - - octave_builtin *find_builtin (const std::string& name); - - jit_function mirror_binary (const jit_function& fn); - - llvm::Function *wrap_complex (llvm::Function *wrap); - - static llvm::Value *pack_complex (llvm::IRBuilderD& bld, - llvm::Value *cplx); - - static llvm::Value *unpack_complex (llvm::IRBuilderD& bld, - llvm::Value *result); - - llvm::Value *complex_real (llvm::Value *cx); - - llvm::Value *complex_real (llvm::Value *cx, llvm::Value *real); - - llvm::Value *complex_imag (llvm::Value *cx); - - llvm::Value *complex_imag (llvm::Value *cx, llvm::Value *imag); - - llvm::Value *complex_new (llvm::Value *real, llvm::Value *imag); - - void create_int (size_t nbits); - - jit_type *intN (size_t nbits) const; - - static jit_typeinfo *instance; - - llvm::Module *module; - llvm::ExecutionEngine *engine; - int next_id; - - llvm::GlobalVariable *lerror_state; - llvm::GlobalVariable *loctave_interrupt_state; - - llvm::Type *sig_atomic_type; - - std::vector<jit_type*> id_to_type; - jit_type *any; - jit_type *matrix; - jit_type *scalar; - jit_type *scalar_ptr; // a fake type for interfacing with C++ - jit_type *any_ptr; // a fake type for interfacing with C++ - jit_type *range; - jit_type *string; - jit_type *boolean; - jit_type *index; - jit_type *complex; - jit_type *unknown_function; - std::map<size_t, jit_type *> ints; - std::map<std::string, jit_type *> builtins; - - llvm::StructType *complex_ret; - - std::vector<jit_operation> binary_ops; - std::vector<jit_operation> unary_ops; - jit_operation grab_fn; - jit_operation release_fn; - jit_operation destroy_fn; - jit_operation print_fn; - jit_operation for_init_fn; - jit_operation for_check_fn; - jit_operation for_index_fn; - jit_operation logically_true_fn; - jit_operation make_range_fn; - jit_paren_subsref paren_subsref_fn; - jit_paren_subsasgn paren_subsasgn_fn; - jit_operation end1_fn; - jit_operation end_fn; - jit_operation create_undef_fn; - - jit_function any_call; - - // type id -> cast function TO that type - std::vector<jit_operation> casts; - - // type id -> identity function - std::vector<jit_function> identities; - - llvm::IRBuilderD& builder; -}; - -#endif -#endif
--- a/libinterp/corefcn/jit-util.cc Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,51 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -// defines required by llvm -#define __STDC_LIMIT_MACROS -#define __STDC_CONSTANT_MACROS - -#if defined (HAVE_CONFIG_H) -# include "config.h" -#endif - -#if defined (HAVE_LLVM) - -#if defined (HAVE_LLVM_IR_FUNCTION_H) -# include <llvm/IR/Value.h> -#else -# include <llvm/Value.h> -#endif - -#include <llvm/Support/raw_os_ostream.h> - -std::ostream& -operator<< (std::ostream& os, const llvm::Value& v) -{ - llvm::raw_os_ostream llvm_out (os); - v.print (llvm_out); - return os; -} - -#endif
--- a/libinterp/corefcn/jit-util.h Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,219 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -// Some utility classes and functions used throughout jit - -#if ! defined (octave_jit_util_h) -#define octave_jit_util_h 1 - -#include "octave-config.h" - -#if defined (HAVE_LLVM) - -#include <stdexcept> - -#if defined (HAVE_LLVM_IR_DATALAYOUT_H) || defined (HAVE_LLVM_DATALAYOUT_H) -# define HAVE_LLVM_DATALAYOUT -#endif - -// we don't want to include llvm headers here, as they require -// __STDC_LIMIT_MACROS and __STDC_CONSTANT_MACROS be defined in the entire -// compilation unit -namespace llvm -{ - class Value; - class Module; -#if defined (LEGACY_PASSMANAGER) - namespace legacy - { - class FunctionPassManager; - class PassManager; - } -#else - class FunctionPassManager; - class PassManager; -#endif - class ExecutionEngine; - class Function; - class BasicBlock; - class LLVMContext; - class Type; - class StructType; - class Twine; - class GlobalVariable; - class TerminatorInst; - class PHINode; - - class ConstantFolder; - - template <bool preserveNames> - class IRBuilderDefaultInserter; - - template <bool preserveNames, typename T, typename Inserter> - class IRBuilder; - -typedef IRBuilder<true, ConstantFolder, IRBuilderDefaultInserter<true> > -IRBuilderD; -} - -class octave_base_value; -class octave_builtin; -class octave_value; -class tree; -class tree_expression; - -// thrown when we should give up on JIT and interpret -class jit_fail_exception : public std::runtime_error -{ -public: - jit_fail_exception (void) : std::runtime_error ("unknown"), mknown (false) { } - jit_fail_exception (const std::string& reason) : std::runtime_error (reason), - mknown (true) - { } - - bool known (void) const { return mknown; } -private: - bool mknown; -}; - -// llvm doesn't provide this, and it's really useful for debugging -std::ostream& operator<< (std::ostream& os, const llvm::Value& v); - -template <typename HOLDER_T, typename SUB_T> -class jit_internal_node; - -// jit_internal_list and jit_internal_node implement generic embedded doubly -// linked lists. List items extend from jit_internal_list, and can be placed -// in nodes of type jit_internal_node. We use CRTP twice. -template <typename LIST_T, typename NODE_T> -class -jit_internal_list -{ - friend class jit_internal_node<LIST_T, NODE_T>; -public: - jit_internal_list (void) : use_head (0), use_tail (0), muse_count (0) { } - - virtual ~jit_internal_list (void) - { - while (use_head) - use_head->stash_value (0); - } - - NODE_T *first_use (void) const { return use_head; } - - size_t use_count (void) const { return muse_count; } -private: - NODE_T *use_head; - NODE_T *use_tail; - size_t muse_count; -}; - -// a node for internal linked lists -template <typename LIST_T, typename NODE_T> -class -jit_internal_node -{ -public: - typedef jit_internal_list<LIST_T, NODE_T> jit_ilist; - - jit_internal_node (void) : mvalue (0), mnext (0), mprev (0) { } - - ~jit_internal_node (void) { remove (); } - - LIST_T *value (void) const { return mvalue; } - - void stash_value (LIST_T *avalue) - { - remove (); - - mvalue = avalue; - - if (mvalue) - { - jit_ilist *ilist = mvalue; - NODE_T *sthis = static_cast<NODE_T *> (this); - if (ilist->use_head) - { - ilist->use_tail->mnext = sthis; - mprev = ilist->use_tail; - } - else - ilist->use_head = sthis; - - ilist->use_tail = sthis; - ++ilist->muse_count; - } - } - - NODE_T *next (void) const { return mnext; } - - NODE_T *prev (void) const { return mprev; } -private: - void remove () - { - if (mvalue) - { - jit_ilist *ilist = mvalue; - if (mprev) - mprev->mnext = mnext; - else - // we are the use_head - ilist->use_head = mnext; - - if (mnext) - mnext->mprev = mprev; - else - // we are the use tail - ilist->use_tail = mprev; - - mnext = mprev = 0; - --ilist->muse_count; - mvalue = 0; - } - } - - LIST_T *mvalue; - NODE_T *mnext; - NODE_T *mprev; -}; - -// Use like: isa<jit_phi> (value) -// basically just a short cut type typing dyanmic_cast. -template <typename T, typename U> -bool isa (U *value) -{ - return dynamic_cast<T *> (value); -} - -#define JIT_ASSIGN_ARG(i) the_args[i] = arg ## i; -#define JIT_EXPAND(ret, fname, type, isconst, N) \ - ret fname (JIT_PARAM_ARGS OCT_MAKE_DECL_LIST (type, arg, N)) isconst \ - { \ - std::vector<type> the_args (N); \ - OCT_ITERATE_MACRO (JIT_ASSIGN_ARG, N); \ - return fname (JIT_PARAMS the_args); \ - } - -#endif -#endif
--- a/libinterp/corefcn/module.mk Tue Jan 31 17:42:29 2017 -0500 +++ b/libinterp/corefcn/module.mk Tue Jan 31 17:43:58 2017 -0500 @@ -16,12 +16,6 @@ DIRSTAMP_FILES += libinterp/corefcn/$(octave_dirstamp) -JIT_INC = \ - libinterp/corefcn/jit-util.h \ - libinterp/corefcn/jit-typeinfo.h \ - libinterp/corefcn/jit-ir.h \ - libinterp/corefcn/pt-jit.h - COREFCN_INC = \ libinterp/corefcn/base-text-renderer.h \ libinterp/corefcn/Cell.h \ @@ -97,14 +91,7 @@ libinterp/corefcn/xdiv.h \ libinterp/corefcn/xnorm.h \ libinterp/corefcn/xpow.h \ - libinterp/corefcn/zfstream.h \ - $(JIT_INC) - -JIT_SRC = \ - libinterp/corefcn/jit-util.cc \ - libinterp/corefcn/jit-typeinfo.cc \ - libinterp/corefcn/jit-ir.cc \ - libinterp/corefcn/pt-jit.cc + libinterp/corefcn/zfstream.h NOINSTALL_COREFCN_INC = \ libinterp/corefcn/oct-hdf5.h \ @@ -258,7 +245,6 @@ libinterp/corefcn/xnorm.cc \ libinterp/corefcn/xpow.cc \ libinterp/corefcn/zfstream.cc \ - $(JIT_SRC) \ $(NOINSTALL_COREFCN_INC) ## Special rules for sources which must be built before rest of compilation.
--- a/libinterp/corefcn/pt-jit.cc Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2607 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -#define __STDC_LIMIT_MACROS -#define __STDC_CONSTANT_MACROS - -#if defined (HAVE_CONFIG_H) -# include "config.h" -#endif - -#include "debug.h" -#include "defun.h" -#include "errwarn.h" -#include "ov.h" -#include "pt-all.h" -#include "pt-jit.h" -#include "sighandlers.h" -#include "symtab.h" -#include "variables.h" - -#if defined (HAVE_LLVM) - -static bool Vdebug_jit = false; - -static bool Vjit_enable = false; - -static int Vjit_startcnt = 1000; - -static int Vjit_failcnt = 0; - -#include <llvm/Analysis/CallGraph.h> -#include <llvm/Analysis/Passes.h> - -#if defined (HAVE_LLVM_IR_VERIFIER_H) -# include <llvm/IR/Verifier.h> -#else -# include <llvm/Analysis/Verifier.h> -#endif - -#include <llvm/Bitcode/ReaderWriter.h> -#include <llvm/ExecutionEngine/ExecutionEngine.h> -#include <llvm/ExecutionEngine/JIT.h> - -#if defined (LEGACY_PASSMANAGER) -# include <llvm/IR/LegacyPassManager.h> -#else -# include <llvm/PassManager.h> -#endif - -#if defined (HAVE_LLVM_IR_FUNCTION_H) -# include <llvm/IR/LLVMContext.h> -# include <llvm/IR/Module.h> -#else -# include <llvm/LLVMContext.h> -# include <llvm/Module.h> -#endif - -#if defined (HAVE_LLVM_SUPPORT_IRBUILDER_H) -# include <llvm/Support/IRBuilder.h> -#elif defined(HAVE_LLVM_IR_IRBUILDER_H) -# include <llvm/IR/IRBuilder.h> -#else -# include <llvm/IRBuilder.h> -#endif - -#include <llvm/Support/raw_os_ostream.h> -#include <llvm/Support/TargetSelect.h> - -#if defined (HAVE_LLVM_IR_DATALAYOUT_H) -# include <llvm/IR/DataLayout.h> -#elif defined(HAVE_LLVM_DATALAYOUT_H) -# include <llvm/DataLayout.h> -#else -# include <llvm/Target/TargetData.h> -#endif - -#include <llvm/Transforms/IPO.h> -#include <llvm/Transforms/Scalar.h> - -static llvm::IRBuilder<> builder (llvm::getGlobalContext ()); - -static llvm::LLVMContext& context = llvm::getGlobalContext (); - -// -------------------- jit_break_exception -------------------- - -// jit_break is thrown whenever a branch we are converting has only breaks or -// continues. This is because all code that follows a break or continue -// is dead. -class jit_break_exception : public std::exception -{ }; - -// -------------------- jit_convert -------------------- -jit_convert::jit_convert (tree &tee, jit_type *for_bounds) - : converting_function (false) -{ - initialize (symbol_table::current_scope ()); - - if (for_bounds) - create_variable (next_for_bounds (false), for_bounds); - - try - { - visit (tee); - } - catch (const jit_break_exception&) - { } - - // breaks must have been handled by the top level loop - assert (breaks.empty ()); - assert (continues.empty ()); - - block->append (factory.create<jit_branch> (final_block)); - blocks.push_back (final_block); - - for (variable_map::iterator iter = vmap.begin (); iter != vmap.end (); ++iter) - { - jit_variable *var = iter->second; - const std::string& name = var->name (); - if (name.size () && name[0] != '#') - final_block->append (factory.create<jit_store_argument> (var)); - } - - final_block->append (factory.create<jit_return> ()); -} - -jit_convert::jit_convert (octave_user_function& fcn, - const std::vector<jit_type *>& args) - : converting_function (true) -{ - initialize (fcn.scope ()); - - tree_parameter_list *plist = fcn.parameter_list (); - tree_parameter_list *rlist = fcn.return_list (); - if (plist && plist->takes_varargs ()) - throw jit_fail_exception ("varags not supported"); - - if (rlist && (rlist->size () > 1 || rlist->takes_varargs ())) - throw jit_fail_exception ("multiple returns not supported"); - - if (plist) - { - tree_parameter_list::iterator piter = plist->begin (); - for (size_t i = 0; i < args.size (); ++i, ++piter) - { - if (piter == plist->end ()) - throw jit_fail_exception ("Too many parameter to function"); - - tree_decl_elt *elt = *piter; - std::string name = elt->name (); - create_variable (name, args[i]); - } - } - - jit_value *return_value = 0; - bool all_breaking = false; - if (fcn.is_special_expr ()) - { - tree_expression *expr = fcn.special_expr (); - if (expr) - { - jit_variable *retvar = get_variable ("#return"); - jit_value *retval = 0; - try - { - retval = visit (expr); - } - catch (const jit_break_exception&) - { } - - if (breaks.size () || continues.size ()) - throw jit_fail_exception ("break/continue not supported in " - "anonymous functions"); - - block->append (factory.create<jit_assign> (retvar, retval)); - return_value = retvar; - } - } - else - { - try - { - visit_statement_list (*fcn.body ()); - } - catch (const jit_break_exception&) - { - all_breaking = true; - } - - // the user may use break or continue to exit the function - finish_breaks (final_block, continues); - finish_breaks (final_block, breaks); - } - - if (! all_breaking) - block->append (factory.create<jit_branch> (final_block)); - - blocks.push_back (final_block); - block = final_block; - - if (! return_value && rlist && rlist->size () == 1) - { - tree_decl_elt *elt = rlist->front (); - return_value = get_variable (elt->name ()); - } - - // FIXME: We should use live range analysis to delete variables where needed. - // For now we just delete everything at the end of the function. - for (variable_map::iterator iter = vmap.begin (); iter != vmap.end (); ++iter) - { - if (iter->second != return_value) - { - jit_call *call; - call = factory.create<jit_call> (&jit_typeinfo::destroy, - iter->second); - final_block->append (call); - } - } - - if (return_value) - final_block->append (factory.create<jit_return> (return_value)); - else - final_block->append (factory.create<jit_return> ()); -} - -void -jit_convert::visit_anon_fcn_handle (tree_anon_fcn_handle&) -{ - throw jit_fail_exception ("No visit_anon_fcn_handle implementation"); -} - -void -jit_convert::visit_argument_list (tree_argument_list&) -{ - throw jit_fail_exception ("No visit_argument_list implementation"); -} - -void -jit_convert::visit_binary_expression (tree_binary_expression& be) -{ - if (be.op_type () >= octave_value::num_binary_ops) - { - tree_boolean_expression *boole; - boole = dynamic_cast<tree_boolean_expression *> (&be); - assert (boole); - bool is_and = boole->op_type () == tree_boolean_expression::bool_and; - - std::string short_name = next_shortcircut_result (); - jit_variable *short_result = factory.create<jit_variable> (short_name); - vmap[short_name] = short_result; - - jit_block *done = factory.create<jit_block> (block->name ()); - tree_expression *lhs = be.lhs (); - jit_value *lhsv = visit (lhs); - lhsv = create_checked (&jit_typeinfo::logically_true, lhsv); - - jit_block *short_early = factory.create<jit_block> ("short_early"); - blocks.push_back (short_early); - - jit_block *short_cont = factory.create<jit_block> ("short_cont"); - - if (is_and) - block->append (factory.create<jit_cond_branch> (lhsv, short_cont, - short_early)); - else - block->append (factory.create<jit_cond_branch> (lhsv, short_early, - short_cont)); - - block = short_early; - - jit_value *early_result = factory.create<jit_const_bool> (! is_and); - block->append (factory.create<jit_assign> (short_result, early_result)); - block->append (factory.create<jit_branch> (done)); - - blocks.push_back (short_cont); - block = short_cont; - - tree_expression *rhs = be.rhs (); - jit_value *rhsv = visit (rhs); - rhsv = create_checked (&jit_typeinfo::logically_true, rhsv); - block->append (factory.create<jit_assign> (short_result, rhsv)); - block->append (factory.create<jit_branch> (done)); - - blocks.push_back (done); - block = done; - result = short_result; - } - else - { - tree_expression *lhs = be.lhs (); - jit_value *lhsv = visit (lhs); - - tree_expression *rhs = be.rhs (); - jit_value *rhsv = visit (rhs); - - const jit_operation& fn = jit_typeinfo::binary_op (be.op_type ()); - result = create_checked (fn, lhsv, rhsv); - } -} - -void -jit_convert::visit_break_command (tree_break_command&) -{ - breaks.push_back (block); - throw jit_break_exception (); -} - -void -jit_convert::visit_colon_expression (tree_colon_expression& expr) -{ - // in the futher we need to add support for classes and deal with rvalues - jit_value *base = visit (expr.base ()); - jit_value *limit = visit (expr.limit ()); - jit_value *increment; - tree_expression *tinc = expr.increment (); - - if (tinc) - increment = visit (tinc); - else - increment = factory.create<jit_const_scalar> (1); - - result = block->append (factory.create<jit_call> (jit_typeinfo::make_range, - base, limit, increment)); -} - -void -jit_convert::visit_continue_command (tree_continue_command&) -{ - continues.push_back (block); - throw jit_break_exception (); -} - -void -jit_convert::visit_global_command (tree_global_command&) -{ - throw jit_fail_exception ("No visit_global_command implemenation"); -} - -void -jit_convert::visit_persistent_command (tree_persistent_command&) -{ - throw jit_fail_exception ("No visit_persistent_command implementation"); -} - -void -jit_convert::visit_decl_elt (tree_decl_elt&) -{ - throw jit_fail_exception ("No visit_decl_elt implementation"); -} - -void -jit_convert::visit_decl_init_list (tree_decl_init_list&) -{ - throw jit_fail_exception ("No visit_decl_init_list implementation"); -} - -void -jit_convert::visit_simple_for_command (tree_simple_for_command& cmd) -{ - // Note we do an initial check to see if the loop will run atleast once. - // This allows us to get better type inference bounds on variables defined - // and used only inside the for loop (e.g., the index variable) - - // If we are a nested for loop we need to store the previous breaks - octave::unwind_protect frame; - frame.protect_var (breaks); - frame.protect_var (continues); - breaks.clear (); - continues.clear (); - - // we need a variable for our iterator, because it is used in multiple blocks - std::string iter_name = next_iterator (); - jit_variable *iterator = factory.create<jit_variable> (iter_name); - factory.create<jit_variable> (iter_name); - vmap[iter_name] = iterator; - - jit_block *body = factory.create<jit_block> ("for_body"); - jit_block *tail = factory.create<jit_block> ("for_tail"); - - // do control expression, iter init, and condition check in prev_block (block) - // if we are the top level for loop, the bounds is an input argument. - jit_value *control = find_variable (next_for_bounds ()); - if (! control) - control = visit (cmd.control_expr ()); - jit_call *init_iter = factory.create<jit_call> (jit_typeinfo::for_init, - control); - block->append (init_iter); - block->append (factory.create<jit_assign> (iterator, init_iter)); - - jit_call *check = factory.create<jit_call> (jit_typeinfo::for_check, control, - iterator); - block->append (check); - block->append (factory.create<jit_cond_branch> (check, body, tail)); - - blocks.push_back (body); - block = body; - - // compute the syntactical iterator - jit_call *idx_rhs = factory.create<jit_call> (jit_typeinfo::for_index, - control, iterator); - block->append (idx_rhs); - do_assign (cmd.left_hand_side (), idx_rhs); - - // do loop - tree_statement_list *pt_body = cmd.body (); - bool all_breaking = false; - try - { - pt_body->accept (*this); - } - catch (const jit_break_exception&) - { - if (continues.empty ()) - { - // WTF are you doing user? Every branch was a break, why did you have - // a loop??? Users are silly people... - finish_breaks (tail, breaks); - blocks.push_back (tail); - block = tail; - return; - } - - all_breaking = true; - } - - // check our condition, continues jump to this block - jit_block *check_block = factory.create<jit_block> ("for_check"); - blocks.push_back (check_block); - - jit_block *interrupt_check = factory.create<jit_block> ("for_interrupt"); - blocks.push_back (interrupt_check); - - if (! all_breaking) - block->append (factory.create<jit_branch> (check_block)); - finish_breaks (check_block, continues); - - block = check_block; - const jit_operation& add_fn = jit_typeinfo::binary_op (octave_value::op_add); - jit_value *one = factory.create<jit_const_index> (1); - jit_call *iter_inc = factory.create<jit_call> (add_fn, iterator, one); - block->append (iter_inc); - block->append (factory.create<jit_assign> (iterator, iter_inc)); - check = block->append (factory.create<jit_call> (jit_typeinfo::for_check, - control, iterator)); - block->append (factory.create<jit_cond_branch> (check, interrupt_check, - tail)); - - block = interrupt_check; - jit_error_check *ec - = factory.create<jit_error_check> (jit_error_check::var_interrupt, - body, final_block); - block->append (ec); - - // breaks will go to our tail - blocks.push_back (tail); - finish_breaks (tail, breaks); - block = tail; -} - -void -jit_convert::visit_complex_for_command (tree_complex_for_command&) -{ - throw jit_fail_exception ("No visit_complex_for_command implementation"); -} - -void -jit_convert::visit_octave_user_script (octave_user_script&) -{ - throw jit_fail_exception ("No visit_octave_user_script implementation"); -} - -void -jit_convert::visit_octave_user_function (octave_user_function&) -{ - throw jit_fail_exception ("No visit_octave_user_function implementation"); -} - -void -jit_convert::visit_octave_user_function_header (octave_user_function&) -{ - throw jit_fail_exception ("No visit_octave_user_function_header implementation"); -} - -void -jit_convert::visit_octave_user_function_trailer (octave_user_function&) -{ - throw jit_fail_exception ("No visit_octave_user_function_trailer implementation"); -} - -void -jit_convert::visit_function_def (tree_function_def&) -{ - throw jit_fail_exception ("No visit_function_def implementation"); -} - -void -jit_convert::visit_identifier (tree_identifier& ti) -{ - if (ti.has_magic_end ()) - { - if (! end_context.size ()) - throw jit_fail_exception ("Illegal end"); - result = block->append (factory.create<jit_magic_end> (end_context)); - } - else - { - jit_variable *var = get_variable (ti.name ()); - jit_instruction *instr; - instr = factory.create<jit_call> (&jit_typeinfo::grab, var); - result = block->append (instr); - } -} - -void -jit_convert::visit_if_clause (tree_if_clause&) -{ - throw jit_fail_exception ("No visit_if_clause implementation"); -} - -void -jit_convert::visit_if_command (tree_if_command& cmd) -{ - tree_if_command_list *lst = cmd.cmd_list (); - assert (lst); // jwe: Can this be null? - lst->accept (*this); -} - -void -jit_convert::visit_if_command_list (tree_if_command_list& lst) -{ - tree_if_clause *last = lst.back (); - size_t last_else = static_cast<size_t> (last->is_else_clause ()); - - // entry_blocks represents the block you need to enter in order to execute - // the condition check for the ith clause. For the else, it is simple the - // else body. If there is no else body, then it is padded with the tail. - std::vector<jit_block *> entry_blocks (lst.size () + 1 - last_else); - entry_blocks[0] = block; - - // we need to construct blocks first, because they have jumps to each other. - tree_if_command_list::iterator iter = lst.begin (); - ++iter; - for (size_t i = 1; iter != lst.end (); ++iter, ++i) - { - tree_if_clause *tic = *iter; - if (tic->is_else_clause ()) - entry_blocks[i] = factory.create<jit_block> ("else"); - else - entry_blocks[i] = factory.create<jit_block> ("ifelse_cond"); - } - - jit_block *tail = factory.create<jit_block> ("if_tail"); - if (! last_else) - entry_blocks[entry_blocks.size () - 1] = tail; - - // each branch in the if statement will have different breaks/continues - block_list current_breaks = breaks; - block_list current_continues = continues; - breaks.clear (); - continues.clear (); - - size_t num_incomming = 0; // number of incomming blocks to our tail - iter = lst.begin (); - for (size_t i = 0; iter != lst.end (); ++iter, ++i) - { - tree_if_clause *tic = *iter; - block = entry_blocks[i]; - assert (block); - - if (i) // the first block is prev_block, so it has already been added - blocks.push_back (entry_blocks[i]); - - if (! tic->is_else_clause ()) - { - tree_expression *expr = tic->condition (); - jit_value *cond = visit (expr); - jit_call *check = create_checked (&jit_typeinfo::logically_true, - cond); - jit_block *body = factory.create<jit_block> (i == 0 ? "if_body" - : "ifelse_body"); - blocks.push_back (body); - - jit_instruction *br = factory.create<jit_cond_branch> (check, body, - entry_blocks[i + 1]); - block->append (br); - block = body; - } - - tree_statement_list *stmt_lst = tic->commands (); - assert (stmt_lst); // jwe: Can this be null? - - try - { - stmt_lst->accept (*this); - ++num_incomming; - block->append (factory.create<jit_branch> (tail)); - } - catch (const jit_break_exception&) - { } - - current_breaks.splice (current_breaks.end (), breaks); - current_continues.splice (current_continues.end (), continues); - } - - breaks.splice (breaks.end (), current_breaks); - continues.splice (continues.end (), current_continues); - - if (num_incomming || ! last_else) - { - blocks.push_back (tail); - block = tail; - } - else - // every branch broke, so we don't have a tail - throw jit_break_exception (); -} - -void -jit_convert::visit_index_expression (tree_index_expression& exp) -{ - result = resolve (exp); -} - -void -jit_convert::visit_matrix (tree_matrix&) -{ - throw jit_fail_exception ("No visit_matrix implementation"); -} - -void -jit_convert::visit_cell (tree_cell&) -{ - throw jit_fail_exception ("No visit_cell implementation"); -} - -void -jit_convert::visit_multi_assignment (tree_multi_assignment&) -{ - throw jit_fail_exception ("No visit_multi_assignment implementation"); -} - -void -jit_convert::visit_no_op_command (tree_no_op_command&) -{ - throw jit_fail_exception ("No visit_no_op_command implementation"); -} - -void -jit_convert::visit_constant (tree_constant& tc) -{ - octave_value v = tc.rvalue1 (); - jit_type *ty = jit_typeinfo::type_of (v); - - if (ty == jit_typeinfo::get_scalar ()) - { - double dv = v.double_value (); - result = factory.create<jit_const_scalar> (dv); - } - else if (ty == jit_typeinfo::get_range ()) - { - Range rv = v.range_value (); - result = factory.create<jit_const_range> (rv); - } - else if (ty == jit_typeinfo::get_complex ()) - { - Complex cv = v.complex_value (); - result = factory.create<jit_const_complex> (cv); - } - else - throw jit_fail_exception ("Unknown constant"); -} - -void -jit_convert::visit_fcn_handle (tree_fcn_handle&) -{ - throw jit_fail_exception ("No visit_fcn_handle implementation"); -} - -void -jit_convert::visit_funcall (tree_funcall&) -{ - throw jit_fail_exception (); -} - -void -jit_convert::visit_parameter_list (tree_parameter_list&) -{ - throw jit_fail_exception ("No visit_parameter_list implementation"); -} - -void -jit_convert::visit_postfix_expression (tree_postfix_expression& tpe) -{ - octave_value::unary_op etype = tpe.op_type (); - tree_expression *operand = tpe.operand (); - jit_value *operandv = visit (operand); - - const jit_operation& fn = jit_typeinfo::unary_op (etype); - result = create_checked (fn, operandv); - - if (etype == octave_value::op_incr || etype == octave_value::op_decr) - { - jit_value *ret = create_checked (&jit_typeinfo::grab, operandv); - do_assign (operand, result); - result = ret; - } -} - -void -jit_convert::visit_prefix_expression (tree_prefix_expression& tpe) -{ - octave_value::unary_op etype = tpe.op_type (); - tree_expression *operand = tpe.operand (); - const jit_operation& fn = jit_typeinfo::unary_op (etype); - result = create_checked (fn, visit (operand)); - - if (etype == octave_value::op_incr || etype == octave_value::op_decr) - do_assign (operand, result); -} - -void -jit_convert::visit_return_command (tree_return_command&) -{ - throw jit_fail_exception ("No visit_return_command implementation"); -} - -void -jit_convert::visit_return_list (tree_return_list&) -{ - throw jit_fail_exception ("No visit_return_list implementation"); -} - -void -jit_convert::visit_simple_assignment (tree_simple_assignment& tsa) -{ - tree_expression *rhs = tsa.right_hand_side (); - jit_value *rhsv = visit (rhs); - octave_value::assign_op op = tsa.op_type (); - - if (op != octave_value::op_asn_eq) - { - // Do the equivalent binary operation, then assign. - // This is always correct, but it isn't always optimal. - tree_expression *lhs = tsa.left_hand_side (); - jit_value *lhsv = visit (lhs); - octave_value::binary_op bop = octave_value::assign_op_to_binary_op (op); - const jit_operation& fn = jit_typeinfo::binary_op (bop); - rhsv = create_checked (fn, lhsv, rhsv); - } - - result = do_assign (tsa.left_hand_side (), rhsv); -} - -void -jit_convert::visit_statement (tree_statement& stmt) -{ - tree_command *cmd = stmt.command (); - tree_expression *expr = stmt.expression (); - - if (cmd) - visit (cmd); - else - { - // stolen from octave::tree_evaluator::visit_statement - bool do_bind_ans = false; - - if (expr->is_identifier ()) - { - tree_identifier *id = dynamic_cast<tree_identifier *> (expr); - - do_bind_ans = (! id->is_variable ()); - } - else - do_bind_ans = (! expr->is_assignment_expression ()); - - jit_value *expr_result = visit (expr); - - if (do_bind_ans) - do_assign ("ans", expr_result, expr->print_result ()); - else if (expr->is_identifier () && expr->print_result ()) - { - // FIXME: ugly hack, we need to come up with a way to pass - // nargout to visit_identifier - const jit_operation& fn = jit_typeinfo::print_value (); - jit_const_string *name = factory.create<jit_const_string> - (expr->name ()); - block->append (factory.create<jit_call> (fn, name, expr_result)); - } - } -} - -void -jit_convert::visit_statement_list (tree_statement_list& lst) -{ - for (tree_statement_list::iterator iter = lst.begin (); iter != lst.end(); - ++iter) - { - tree_statement *elt = *iter; - // jwe: Can this ever be null? - assert (elt); - elt->accept (*this); - } -} - -void -jit_convert::visit_switch_case (tree_switch_case&) -{ - throw jit_fail_exception ("No visit_switch_case implementation"); -} - -void -jit_convert::visit_switch_case_list (tree_switch_case_list&) -{ - throw jit_fail_exception ("No visit_switch_case_list implementation"); -} - -void -jit_convert::visit_switch_command (tree_switch_command& cmd) -{ - tree_switch_case_list *lst = cmd.case_list (); - - // always visit switch expression - tree_expression *expr = cmd.switch_value (); - assert (expr && "Switch value can not be null"); - jit_value *value = visit (expr); - assert (value); - - size_t case_blocks_num = lst->size (); - - if (! case_blocks_num) // there's nothing to do - return; - - // check for otherwise, it's interpreted as last 'else' condition - size_t has_otherwise = 0; - tree_switch_case *last = lst->back (); - if (last->is_default_case ()) - has_otherwise = 1; - - std::vector<jit_block *> entry_blocks (case_blocks_num + 1 - has_otherwise); - - // the first entry point is always the actual block. Afterward, new blocks - // are created for every case and the otherwise branch - entry_blocks[0] = block; - for (size_t i = 1; i < case_blocks_num; ++i) - entry_blocks[i] = factory.create<jit_block> ("case_cond"); - - jit_block *tail = factory.create<jit_block> ("switch_tail"); - - // if there's no otherwise branch, the 'else' of the last branch - // has to point to the tail - if (! has_otherwise) - entry_blocks[entry_blocks.size()-1] = tail; - - // each branch in the case statement will have different breaks/continues - block_list current_breaks = breaks; - block_list current_continues = continues; - breaks.clear (); - continues.clear (); - - size_t num_incomming = 0; // number of incomming blocks to our tail - - tree_switch_case_list::iterator iter = lst->begin (); - for (size_t i = 0; i < case_blocks_num; ++iter, ++i) - { - tree_switch_case *twc = *iter; - block = entry_blocks[i]; // case_cond - assert (block); - - if (i) - blocks.push_back (entry_blocks[i]); // first block already pushed - - if (! twc->is_default_case ()) - { - // compare result of switch expression with actual case label - tree_expression *te = twc->case_label (); - jit_value *label = visit (te); - assert(label); - - const jit_operation& fn = jit_typeinfo::binary_op (octave_value::op_eq); - jit_value *cond = create_checked (fn, value, label); - assert(cond); - - jit_call *check = create_checked (&jit_typeinfo::logically_true, - cond); - - jit_block *body = factory.create<jit_block> ("case_body"); - blocks.push_back (body); - - block->append (factory.create<jit_cond_branch> (check, body, - entry_blocks[i+1])); - block = body; // case_body - } - - tree_statement_list *stmt_lst = twc->commands (); - assert(stmt_lst); - - try - { - stmt_lst->accept (*this); - num_incomming++; - block->append (factory.create<jit_branch> (tail)); - } - catch (const jit_break_exception&) - { } - - // each branch in the case statement will have different breaks/continues - current_breaks.splice (current_breaks.end (), breaks); - current_continues.splice (current_continues.end (), continues); - } - - // each branch in the case statement will have different breaks/continues - breaks.splice (breaks.end (), current_breaks); - continues.splice (continues.end (), current_continues); - - if (num_incomming || ! has_otherwise) - { - blocks.push_back (tail); - block = tail; // switch_tail - } - else - throw jit_break_exception (); // every branch broke -} - -void -jit_convert::visit_try_catch_command (tree_try_catch_command&) -{ - throw jit_fail_exception ("No visit_try_catch_command implementation"); -} - -void -jit_convert::visit_unwind_protect_command (tree_unwind_protect_command&) -{ - throw jit_fail_exception ("No visit_unwind_protect_command implementation"); -} - -void -jit_convert::visit_while_command (tree_while_command& wc) -{ - octave::unwind_protect frame; - frame.protect_var (breaks); - frame.protect_var (continues); - breaks.clear (); - continues.clear (); - - jit_block *cond_check = factory.create<jit_block> ("while_cond_check"); - block->append (factory.create<jit_branch> (cond_check)); - blocks.push_back (cond_check); - block = cond_check; - - tree_expression *expr = wc.condition (); - assert (expr && "While expression can not be null"); - jit_value *check = visit (expr); - check = create_checked (&jit_typeinfo::logically_true, check); - - jit_block *body = factory.create<jit_block> ("while_body"); - blocks.push_back (body); - - jit_block *tail = factory.create<jit_block> ("while_tail"); - block->append (factory.create<jit_cond_branch> (check, body, tail)); - block = body; - - tree_statement_list *loop_body = wc.body (); - bool all_breaking = false; - if (loop_body) - { - try - { - loop_body->accept (*this); - } - catch (const jit_break_exception&) - { - all_breaking = true; - } - } - - finish_breaks (tail, breaks); - - if (! all_breaking || continues.size ()) - { - jit_block *interrupt_check - = factory.create<jit_block> ("interrupt_check"); - blocks.push_back (interrupt_check); - finish_breaks (interrupt_check, continues); - if (! all_breaking) - block->append (factory.create<jit_branch> (interrupt_check)); - - block = interrupt_check; - jit_error_check *ec - = factory.create<jit_error_check> (jit_error_check::var_interrupt, - cond_check, final_block); - block->append (ec); - } - - blocks.push_back (tail); - block = tail; -} - -void -jit_convert::visit_do_until_command (tree_do_until_command& duc) -{ - octave::unwind_protect frame; - frame.protect_var (breaks); - frame.protect_var (continues); - breaks.clear (); - continues.clear (); - - jit_block *body = factory.create<jit_block> ("do_until_body"); - jit_block *cond_check = factory.create<jit_block> ("do_until_cond_check"); - jit_block *tail = factory.create<jit_block> ("do_until_tail"); - - block->append (factory.create<jit_branch> (body)); - blocks.push_back (body); - block = body; - - tree_statement_list *loop_body = duc.body (); - bool all_breaking = false; - if (loop_body) - { - try - { - loop_body->accept (*this); - } - catch (const jit_break_exception&) - { - all_breaking = true; - } - } - - finish_breaks (tail, breaks); - - if (! all_breaking || continues.size ()) - { - jit_block *interrupt_check - = factory.create<jit_block> ("interrupt_check"); - blocks.push_back (interrupt_check); - finish_breaks (interrupt_check, continues); - if (! all_breaking) - block->append (factory.create<jit_branch> (interrupt_check)); - - block = interrupt_check; - jit_error_check *ec - = factory.create<jit_error_check> (jit_error_check::var_interrupt, - cond_check, final_block); - block->append (ec); - - blocks.push_back (cond_check); - block = cond_check; - - tree_expression *expr = duc.condition (); - assert (expr && "Do-Until expression can not be null"); - jit_value *check = visit (expr); - check = create_checked (&jit_typeinfo::logically_true, check); - - block->append (factory.create<jit_cond_branch> (check, tail, body)); - } - - blocks.push_back (tail); - block = tail; -} - -void -jit_convert::initialize (symbol_table::scope_id s) -{ - scope = s; - iterator_count = 0; - for_bounds_count = 0; - short_count = 0; - jit_instruction::reset_ids (); - - entry_block = factory.create<jit_block> ("body"); - final_block = factory.create<jit_block> ("final"); - blocks.push_back (entry_block); - entry_block->mark_alive (); - block = entry_block; -} - -jit_call * -jit_convert::create_checked_impl (jit_call *ret) -{ - block->append (ret); - - jit_block *normal = factory.create<jit_block> (block->name ()); - jit_error_check *check - = factory.create<jit_error_check> (jit_error_check::var_error_state, ret, - normal, final_block); - block->append (check); - blocks.push_back (normal); - block = normal; - - return ret; -} - -jit_variable * -jit_convert::find_variable (const std::string& vname) const -{ - variable_map::const_iterator iter; - iter = vmap.find (vname); - return iter != vmap.end () ? iter->second : 0; -} - -jit_variable * -jit_convert::get_variable (const std::string& vname) -{ - jit_variable *ret = find_variable (vname); - if (ret) - return ret; - - symbol_table::symbol_record record = symbol_table::find_symbol (vname, scope); - if (record.is_persistent () || record.is_global ()) - throw jit_fail_exception ("Persistent and global not yet supported"); - - if (converting_function) - return create_variable (vname, jit_typeinfo::get_any (), false); - else - { - octave_value val = record.varval (); - if (val.is_undefined ()) - val = symbol_table::find_function (vname); - - jit_type *type = jit_typeinfo::type_of (val); - bounds.push_back (type_bound (type, vname)); - - return create_variable (vname, type); - } -} - -jit_variable * -jit_convert::create_variable (const std::string& vname, jit_type *type, - bool isarg) -{ - jit_variable *var = factory.create<jit_variable> (vname); - - if (isarg) - { - jit_extract_argument *extract; - extract = factory.create<jit_extract_argument> (type, var); - entry_block->prepend (extract); - } - else - { - jit_call *init = factory.create<jit_call> (&jit_typeinfo::create_undef); - jit_assign *assign = factory.create<jit_assign> (var, init); - entry_block->prepend (assign); - entry_block->prepend (init); - } - - return vmap[vname] = var; -} - -std::string -jit_convert::next_name (const char *prefix, size_t& count, bool inc) -{ - std::stringstream ss; - ss << prefix << count; - if (inc) - ++count; - return ss.str (); -} - -jit_instruction * -jit_convert::resolve (tree_index_expression& exp, jit_value *extra_arg, - bool lhs) -{ - std::string type = exp.type_tags (); - if (! (type.size () == 1 && type[0] == '(')) - throw jit_fail_exception ("Unsupported index operation"); - - std::list<tree_argument_list *> args = exp.arg_lists (); - if (args.size () != 1) - throw jit_fail_exception ("Bad number of arguments in " - "tree_index_expression"); - - tree_argument_list *arg_list = args.front (); - if (! arg_list) - throw jit_fail_exception ("null argument list"); - - if (arg_list->size () < 1) - throw jit_fail_exception ("Empty arg_list"); - - tree_expression *tree_object = exp.expression (); - jit_value *object; - if (lhs) - { - tree_identifier *id = dynamic_cast<tree_identifier *> (tree_object); - if (! id) - throw jit_fail_exception ("expected identifier"); - object = get_variable (id->name ()); - } - else - object = visit (tree_object); - - size_t narg = arg_list->size (); - tree_argument_list::iterator iter = arg_list->begin (); - bool have_extra = extra_arg; - std::vector<jit_value *> call_args (narg + 1 + have_extra); - call_args[0] = object; - - for (size_t idx = 0; iter != arg_list->end (); ++idx, ++iter) - { - octave::unwind_protect frame; - frame.add_method (&end_context, - &std::vector<jit_magic_end::context>::pop_back); - - jit_magic_end::context ctx (factory, object, idx, narg); - end_context.push_back (ctx); - call_args[idx + 1] = visit (*iter); - } - - if (extra_arg) - call_args[call_args.size () - 1] = extra_arg; - - const jit_operation& fres = lhs ? jit_typeinfo::paren_subsasgn () - : jit_typeinfo::paren_subsref (); - - return create_checked (fres, call_args); -} - -jit_value * -jit_convert::do_assign (tree_expression *exp, jit_value *rhs, bool artificial) -{ - if (! exp) - throw jit_fail_exception ("NULL lhs in assign"); - - if (isa<tree_identifier> (exp)) - return do_assign (exp->name (), rhs, exp->print_result (), artificial); - else if (tree_index_expression *idx - = dynamic_cast<tree_index_expression *> (exp)) - { - jit_value *new_object = resolve (*idx, rhs, true); - do_assign (idx->expression (), new_object, true); - - // FIXME: Will not work for values that must be release/grabed - return rhs; - } - else - throw jit_fail_exception ("Unsupported assignment"); -} - -jit_value * -jit_convert::do_assign (const std::string& lhs, jit_value *rhs, - bool print, bool artificial) -{ - jit_variable *var = get_variable (lhs); - jit_assign *assign = block->append (factory.create<jit_assign> (var, rhs)); - - if (artificial) - assign->mark_artificial (); - - if (print) - { - const jit_operation& print_fn = jit_typeinfo::print_value (); - jit_const_string *name = factory.create<jit_const_string> (lhs); - block->append (factory.create<jit_call> (print_fn, name, var)); - } - - return var; -} - -jit_value * -jit_convert::visit (tree& tee) -{ - octave::unwind_protect frame; - frame.protect_var (result); - - tee.accept (*this); - return result; -} - -void -jit_convert::finish_breaks (jit_block *dest, const block_list& lst) -{ - for (block_list::const_iterator iter = lst.begin (); iter != lst.end (); - ++iter) - { - jit_block *b = *iter; - b->append (factory.create<jit_branch> (dest)); - } -} - -// -------------------- jit_convert_llvm -------------------- -llvm::Function * -jit_convert_llvm::convert_loop (llvm::Module *module, - const jit_block_list& blocks, - const std::list<jit_value *>& constants) -{ - converting_function = false; - - // for now just init arguments from entry, later we will have to do something - // more interesting - jit_block *entry_block = blocks.front (); - for (jit_block::iterator iter = entry_block->begin (); - iter != entry_block->end (); ++iter) - if (jit_extract_argument *extract - = dynamic_cast<jit_extract_argument *> (*iter)) - argument_vec.push_back (std::make_pair (extract->name (), true)); - - jit_type *any = jit_typeinfo::get_any (); - - // argument is an array of octave_base_value*, or octave_base_value** - llvm::Type *arg_type = any->to_llvm (); // this is octave_base_value* - arg_type = arg_type->getPointerTo (); - llvm::FunctionType *ft; - ft = llvm::FunctionType::get (llvm::Type::getVoidTy (context), arg_type, - false); - function = llvm::Function::Create (ft, llvm::Function::ExternalLinkage, - "foobar", module); - - try - { - prelude = llvm::BasicBlock::Create (context, "prelude", function); - builder.SetInsertPoint (prelude); - - llvm::Value *arg = function->arg_begin (); - for (size_t i = 0; i < argument_vec.size (); ++i) - { - llvm::Value *loaded_arg = builder.CreateConstInBoundsGEP1_32 (arg, i); - arguments[argument_vec[i].first] = loaded_arg; - } - - convert (blocks, constants); - } - catch (const jit_fail_exception& e) - { - function->eraseFromParent (); - throw; - } - - return function; -} - -jit_function -jit_convert_llvm::convert_function (llvm::Module *module, - const jit_block_list& blocks, - const std::list<jit_value *>& constants, - octave_user_function& fcn, - const std::vector<jit_type *>& args) -{ - converting_function = true; - - jit_block *final_block = blocks.back (); - jit_return *ret = dynamic_cast<jit_return *> (final_block->back ()); - assert (ret); - - creating = jit_function (module, jit_convention::internal, - "foobar", ret->result_type (), args); - function = creating.to_llvm (); - - try - { - prelude = creating.new_block ("prelude"); - builder.SetInsertPoint (prelude); - - tree_parameter_list *plist = fcn.parameter_list (); - if (plist) - { - tree_parameter_list::iterator piter = plist->begin (); - tree_parameter_list::iterator pend = plist->end (); - for (size_t i = 0; i < args.size () && piter != pend; ++i, ++piter) - { - tree_decl_elt *elt = *piter; - std::string arg_name = elt->name (); - arguments[arg_name] = creating.argument (builder, i); - } - } - - convert (blocks, constants); - } - catch (const jit_fail_exception& e) - { - function->eraseFromParent (); - throw; - } - - return creating; -} - -void -jit_convert_llvm::convert (const jit_block_list& blocks, - const std::list<jit_value *>& constants) -{ - std::list<jit_block *>::const_iterator biter; - for (biter = blocks.begin (); biter != blocks.end (); ++biter) - { - jit_block *jblock = *biter; - llvm::BasicBlock *block = llvm::BasicBlock::Create (context, - jblock->name (), - function); - jblock->stash_llvm (block); - } - - jit_block *first = *blocks.begin (); - builder.CreateBr (first->to_llvm ()); - - // constants aren't in the IR, we visit those first - for (std::list<jit_value *>::const_iterator iter = constants.begin (); - iter != constants.end (); ++iter) - if (! isa<jit_instruction> (*iter)) - visit (*iter); - - // convert all instructions - for (biter = blocks.begin (); biter != blocks.end (); ++biter) - visit (*biter); - - // now finish phi nodes - for (biter = blocks.begin (); biter != blocks.end (); ++biter) - { - jit_block& block = **biter; - for (jit_block::iterator piter = block.begin (); - piter != block.end () && isa<jit_phi> (*piter); ++piter) - { - jit_instruction *phi = *piter; - finish_phi (static_cast<jit_phi *> (phi)); - } - } -} - -void -jit_convert_llvm::finish_phi (jit_phi *phi) -{ - llvm::PHINode *llvm_phi = phi->to_llvm (); - for (size_t i = 0; i < phi->argument_count (); ++i) - { - llvm::BasicBlock *pred = phi->incomming_llvm (i); - llvm_phi->addIncoming (phi->argument_llvm (i), pred); - } -} - -void -jit_convert_llvm::visit (jit_const_string& cs) -{ - cs.stash_llvm (builder.CreateGlobalStringPtr (cs.value ())); -} - -void -jit_convert_llvm::visit (jit_const_bool& cb) -{ - cb.stash_llvm (llvm::ConstantInt::get (cb.type_llvm (), cb.value ())); -} - -void -jit_convert_llvm::visit (jit_const_scalar& cs) -{ - cs.stash_llvm (llvm::ConstantFP::get (cs.type_llvm (), cs.value ())); -} - -void -jit_convert_llvm::visit (jit_const_complex& cc) -{ - llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); - Complex value = cc.value (); - llvm::Value *real = llvm::ConstantFP::get (scalar_t, value.real ()); - llvm::Value *imag = llvm::ConstantFP::get (scalar_t, value.imag ()); - cc.stash_llvm (jit_typeinfo::create_complex (real, imag)); -} - -void jit_convert_llvm::visit (jit_const_index& ci) -{ - ci.stash_llvm (llvm::ConstantInt::get (ci.type_llvm (), ci.value ())); -} - -void -jit_convert_llvm::visit (jit_const_range& cr) -{ - llvm::StructType *stype = llvm::cast<llvm::StructType>(cr.type_llvm ()); - llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); - llvm::Type *idx = jit_typeinfo::get_index_llvm (); - const jit_range& rng = cr.value (); - - llvm::Constant *constants[4]; - constants[0] = llvm::ConstantFP::get (scalar_t, rng.base); - constants[1] = llvm::ConstantFP::get (scalar_t, rng.limit); - constants[2] = llvm::ConstantFP::get (scalar_t, rng.inc); - constants[3] = llvm::ConstantInt::get (idx, rng.nelem); - - llvm::Value *as_llvm; - as_llvm = llvm::ConstantStruct::get (stype, - llvm::makeArrayRef (constants, 4)); - cr.stash_llvm (as_llvm); -} - -void -jit_convert_llvm::visit (jit_block& b) -{ - llvm::BasicBlock *block = b.to_llvm (); - builder.SetInsertPoint (block); - for (jit_block::iterator iter = b.begin (); iter != b.end (); ++iter) - visit (*iter); -} - -void -jit_convert_llvm::visit (jit_branch& b) -{ - b.stash_llvm (builder.CreateBr (b.successor_llvm ())); -} - -void -jit_convert_llvm::visit (jit_cond_branch& cb) -{ - llvm::Value *cond = cb.cond_llvm (); - llvm::Value *br; - br = builder.CreateCondBr (cond, cb.successor_llvm (0), - cb.successor_llvm (1)); - cb.stash_llvm (br); -} - -void -jit_convert_llvm::visit (jit_call& call) -{ - const jit_function& ol = call.overload (); - - std::vector<jit_value *> args (call.arguments ().size ()); - for (size_t i = 0; i < args.size (); ++i) - args[i] = call.argument (i); - - llvm::Value *ret = ol.call (builder, args); - call.stash_llvm (ret); -} - -void -jit_convert_llvm::visit (jit_extract_argument& extract) -{ - llvm::Value *arg = arguments[extract.name ()]; - assert (arg); - - if (converting_function) - extract.stash_llvm (arg); - else - { - arg = builder.CreateLoad (arg); - - const jit_function& ol = extract.overload (); - extract.stash_llvm (ol.call (builder, arg)); - } -} - -void -jit_convert_llvm::visit (jit_store_argument& store) -{ - const jit_function& ol = store.overload (); - llvm::Value *arg_value = ol.call (builder, store.result ()); - llvm::Value *arg = arguments[store.name ()]; - store.stash_llvm (builder.CreateStore (arg_value, arg)); -} - -void -jit_convert_llvm::visit (jit_return& ret) -{ - jit_value *res = ret.result (); - - if (converting_function) - creating.do_return (builder, res->to_llvm (), false); - else - { - if (res) - builder.CreateRet (res->to_llvm ()); - else - builder.CreateRetVoid (); - } -} - -void -jit_convert_llvm::visit (jit_phi& phi) -{ - // we might not have converted all incoming branches, so we don't - // set incomming branches now - llvm::PHINode *node = llvm::PHINode::Create (phi.type_llvm (), - phi.argument_count ()); - builder.Insert (node); - phi.stash_llvm (node); -} - -void -jit_convert_llvm::visit (jit_variable&) -{ - throw jit_fail_exception ("ERROR: SSA construction should remove all variables"); -} - -void -jit_convert_llvm::visit (jit_error_check& check) -{ - llvm::Value *cond; - - switch (check.check_variable ()) - { - case jit_error_check::var_error_state: - cond = jit_typeinfo::insert_error_check (builder); - break; - case jit_error_check::var_interrupt: - cond = jit_typeinfo::insert_interrupt_check (builder); - break; - default: - panic_impossible (); - } - - llvm::Value *br = builder.CreateCondBr (cond, check.successor_llvm (0), - check.successor_llvm (1)); - check.stash_llvm (br); -} - -void -jit_convert_llvm::visit (jit_assign& assign) -{ - jit_value *new_value = assign.src (); - assign.stash_llvm (new_value->to_llvm ()); - - if (assign.artificial ()) - return; - - jit_value *overwrite = assign.overwrite (); - if (isa<jit_assign_base> (overwrite)) - { - const jit_function& ol = jit_typeinfo::get_release (overwrite->type ()); - if (ol.valid ()) - ol.call (builder, overwrite); - } -} - -void -jit_convert_llvm::visit (jit_argument&) -{ } - -void -jit_convert_llvm::visit (jit_magic_end& me) -{ - const jit_function& ol = me.overload (); - - jit_magic_end::context ctx = me.resolve_context (); - llvm::Value *ret = ol.call (builder, ctx.value, ctx.index, ctx.count); - me.stash_llvm (ret); -} - -// -------------------- jit_infer -------------------- -jit_infer::jit_infer (jit_factory& afactory, jit_block_list& ablocks, - const variable_map& avmap) - : blocks (ablocks), factory (afactory), vmap (avmap) { } - -void -jit_infer::infer (void) -{ - construct_ssa (); - - // initialize the worklist to instructions derived from constants - const std::list<jit_value *>& constants = factory.constants (); - for (std::list<jit_value *>::const_iterator iter = constants.begin (); - iter != constants.end (); ++iter) - append_users (*iter); - - // the entry block terminator may be a regular branch statement - if (entry_block ().terminator ()) - push_worklist (entry_block ().terminator ()); - - // FIXME: Describe algorithm here - while (worklist.size ()) - { - jit_instruction *next = worklist.front (); - worklist.pop_front (); - next->stash_in_worklist (false); - - if (next->infer ()) - { - // terminators need to be handles specially - if (jit_terminator *term = dynamic_cast<jit_terminator *> (next)) - append_users_term (term); - else - append_users (next); - } - } - - remove_dead (); - blocks.label (); - place_releases (); - simplify_phi (); -} - -void -jit_infer::append_users (jit_value *v) -{ - for (jit_use *use = v->first_use (); use; use = use->next ()) - push_worklist (use->user ()); -} - -void -jit_infer::append_users_term (jit_terminator *term) -{ - for (size_t i = 0; i < term->successor_count (); ++i) - { - if (term->alive (i)) - { - jit_block *succ = term->successor (i); - for (jit_block::iterator iter = succ->begin (); - iter != succ->end () && isa<jit_phi> (*iter); ++iter) - push_worklist (*iter); - - jit_terminator *sterm = succ->terminator (); - if (sterm) - push_worklist (sterm); - } - } -} - -void -jit_infer::construct_ssa (void) -{ - blocks.label (); - final_block ().compute_idom (entry_block ()); - entry_block ().compute_df (); - entry_block ().create_dom_tree (); - - // insert phi nodes where needed, this is done on a per variable basis - for (variable_map::const_iterator iter = vmap.begin (); iter != vmap.end (); - ++iter) - { - jit_block::df_set visited, added_phi; - std::list<jit_block *> ssa_worklist; - iter->second->use_blocks (visited); - ssa_worklist.insert (ssa_worklist.begin (), visited.begin (), - visited.end ()); - - while (ssa_worklist.size ()) - { - jit_block *b = ssa_worklist.front (); - ssa_worklist.pop_front (); - - for (jit_block::df_iterator diter = b->df_begin (); - diter != b->df_end (); ++diter) - { - jit_block *dblock = *diter; - if (! added_phi.count (dblock)) - { - jit_phi *phi = factory.create<jit_phi> (iter->second, - dblock->use_count ()); - dblock->prepend (phi); - added_phi.insert (dblock); - } - - if (! visited.count (dblock)) - { - ssa_worklist.push_back (dblock); - visited.insert (dblock); - } - } - } - } - - do_construct_ssa (entry_block (), entry_block ().visit_count ()); -} - -void -jit_infer::do_construct_ssa (jit_block& ablock, size_t avisit_count) -{ - if (ablock.visited (avisit_count)) - return; - - // replace variables with their current SSA value - for (jit_block::iterator iter = ablock.begin (); iter != ablock.end (); - ++iter) - { - jit_instruction *instr = *iter; - instr->construct_ssa (); - instr->push_variable (); - } - - // finish phi nodes of successors - for (size_t i = 0; i < ablock.successor_count (); ++i) - { - jit_block *finish = ablock.successor (i); - - for (jit_block::iterator iter = finish->begin (); - iter != finish->end () && isa<jit_phi> (*iter);) - { - jit_phi *phi = static_cast<jit_phi *> (*iter); - jit_variable *var = phi->dest (); - ++iter; - - if (var->has_top ()) - phi->add_incomming (&ablock, var->top ()); - else - { - // temporaries may have extranious phi nodes which can be removed - assert (! phi->use_count ()); - assert (var->name ().size () && var->name ()[0] == '#'); - phi->remove (); - } - } - } - - for (size_t i = 0; i < ablock.dom_successor_count (); ++i) - do_construct_ssa (*ablock.dom_successor (i), avisit_count); - - ablock.pop_all (); -} - -void -jit_infer::place_releases (void) -{ - std::set<jit_value *> temporaries; - for (jit_block_list::iterator iter = blocks.begin (); iter != blocks.end (); - ++iter) - { - jit_block& ablock = **iter; - if (ablock.id () != jit_block::NO_ID) - { - release_temp (ablock, temporaries); - release_dead_phi (ablock); - } - } -} - -void -jit_infer::push_worklist (jit_instruction *instr) -{ - if (! instr->in_worklist ()) - { - instr->stash_in_worklist (true); - worklist.push_back (instr); - } -} - -void -jit_infer::remove_dead () -{ - jit_block_list::iterator biter; - for (biter = blocks.begin (); biter != blocks.end (); ++biter) - { - jit_block *b = *biter; - if (b->alive ()) - { - for (jit_block::iterator iter = b->begin (); - iter != b->end () && isa<jit_phi> (*iter);) - { - jit_phi *phi = static_cast<jit_phi *> (*iter); - if (phi->prune ()) - iter = b->remove (iter); - else - ++iter; - } - } - } - - for (biter = blocks.begin (); biter != blocks.end ();) - { - jit_block *b = *biter; - if (b->alive ()) - { - // FIXME: A special case for jit_error_check, if we generalize to - // we will need to change! - jit_terminator *term = b->terminator (); - if (term && term->successor_count () == 2 && ! term->alive (0)) - { - jit_block *succ = term->successor (1); - term->remove (); - jit_branch *abreak = factory.create<jit_branch> (succ); - b->append (abreak); - abreak->infer (); - } - - ++biter; - } - else - { - jit_terminator *term = b->terminator (); - if (term) - term->remove (); - biter = blocks.erase (biter); - } - } -} - -void -jit_infer::release_dead_phi (jit_block& ablock) -{ - jit_block::iterator iter = ablock.begin (); - while (iter != ablock.end () && isa<jit_phi> (*iter)) - { - jit_phi *phi = static_cast<jit_phi *> (*iter); - ++iter; - - jit_use *use = phi->first_use (); - if (phi->use_count () == 1 && isa<jit_assign> (use->user ())) - { - // instead of releasing on assign, release on all incomming branches, - // this can get rid of casts inside loops - for (size_t i = 0; i < phi->argument_count (); ++i) - { - jit_value *arg = phi->argument (i); - if (! arg->needs_release ()) - continue; - - jit_block *inc = phi->incomming (i); - jit_block *split = inc->maybe_split (factory, blocks, ablock); - jit_terminator *term = split->terminator (); - jit_call *release - = factory.create<jit_call> (jit_typeinfo::release, arg); - release->infer (); - split->insert_before (term, release); - } - - phi->replace_with (0); - phi->remove (); - } - } -} - -void -jit_infer::release_temp (jit_block& ablock, std::set<jit_value *>& temp) -{ - for (jit_block::iterator iter = ablock.begin (); iter != ablock.end (); - ++iter) - { - jit_instruction *instr = *iter; - - // check for temporaries that require release and live across - // multiple blocks - if (instr->needs_release ()) - { - jit_block *fu_block = instr->first_use_block (); - if (fu_block && fu_block != &ablock && instr->needs_release ()) - temp.insert (instr); - } - - if (isa<jit_call> (instr)) - { - // place releases for temporary arguments - for (size_t i = 0; i < instr->argument_count (); ++i) - { - jit_value *arg = instr->argument (i); - if (! arg->needs_release ()) - continue; - - jit_call *release - = factory.create<jit_call> (&jit_typeinfo::release, arg); - release->infer (); - ablock.insert_after (iter, release); - ++iter; - temp.erase (arg); - } - } - } - - if (! temp.size () || ! isa<jit_error_check> (ablock.terminator ())) - return; - - // FIXME: If we support try/catch or unwind_protect final_block - // may not be the destination - jit_block *split = ablock.maybe_split (factory, blocks, final_block ()); - jit_terminator *term = split->terminator (); - for (std::set<jit_value *>::const_iterator iter = temp.begin (); - iter != temp.end (); ++iter) - { - jit_value *value = *iter; - jit_call *release - = factory.create<jit_call> (&jit_typeinfo::release, value); - split->insert_before (term, release); - release->infer (); - } -} - -void -jit_infer::simplify_phi (void) -{ - for (jit_block_list::iterator biter = blocks.begin (); biter != blocks.end (); - ++biter) - { - jit_block &ablock = **biter; - for (jit_block::iterator iter = ablock.begin (); - iter != ablock.end () && isa<jit_phi> (*iter); ++iter) - simplify_phi (*static_cast<jit_phi *> (*iter)); - } -} - -void -jit_infer::simplify_phi (jit_phi& phi) -{ - jit_block& pblock = *phi.parent (); - const jit_operation& cast_fn = jit_typeinfo::cast (phi.type ()); - jit_variable *dest = phi.dest (); - for (size_t i = 0; i < phi.argument_count (); ++i) - { - jit_value *arg = phi.argument (i); - if (arg->type () != phi.type ()) - { - jit_block *pred = phi.incomming (i); - jit_block *split = pred->maybe_split (factory, blocks, pblock); - jit_terminator *term = split->terminator (); - jit_instruction *cast = factory.create<jit_call> (cast_fn, arg); - jit_assign *assign = factory.create<jit_assign> (dest, cast); - - split->insert_before (term, cast); - split->insert_before (term, assign); - cast->infer (); - assign->infer (); - phi.stash_argument (i, assign); - } - } -} - -// -------------------- tree_jit -------------------- - -tree_jit::tree_jit (void) : module (0), engine (0) -{ } - -tree_jit::~tree_jit (void) -{ } - -bool -tree_jit::execute (tree_simple_for_command& cmd, const octave_value& bounds) -{ - return instance ().do_execute (cmd, bounds); -} - -bool -tree_jit::execute (tree_while_command& cmd) -{ - return instance ().do_execute (cmd); -} - -bool -tree_jit::execute (octave_user_function& fcn, const octave_value_list& args, - octave_value_list& retval) -{ - return instance ().do_execute (fcn, args, retval); -} - -tree_jit& -tree_jit::instance (void) -{ - static tree_jit ret; - return ret; -} - -bool -tree_jit::initialize (void) -{ - if (engine) - return true; - - if (! module) - { - llvm::InitializeNativeTarget (); - module = new llvm::Module ("octave", context); - } - - // sometimes this fails pre main - engine = llvm::ExecutionEngine::createJIT (module); - - if (! engine) - return false; - -#if defined (LEGACY_PASSMANAGER) - module_pass_manager = new llvm::legacy::PassManager (); - pass_manager = new llvm::legacy::FunctionPassManager (module); -#else - module_pass_manager = new llvm::PassManager (); - pass_manager = new llvm::FunctionPassManager (module); -#endif - module_pass_manager->add (llvm::createAlwaysInlinerPass ()); - -#if defined (HAVE_LLVM_DATALAYOUT) - pass_manager->add (new llvm::DataLayout (*engine->getDataLayout ())); -#else - pass_manager->add (new llvm::TargetData (*engine->getTargetData ())); -#endif - pass_manager->add (llvm::createCFGSimplificationPass ()); - pass_manager->add (llvm::createBasicAliasAnalysisPass ()); - pass_manager->add (llvm::createPromoteMemoryToRegisterPass ()); - pass_manager->add (llvm::createInstructionCombiningPass ()); - pass_manager->add (llvm::createReassociatePass ()); - pass_manager->add (llvm::createGVNPass ()); - pass_manager->add (llvm::createCFGSimplificationPass ()); - pass_manager->doInitialization (); - - jit_typeinfo::initialize (module, engine); - - return true; -} - -bool -tree_jit::do_execute (tree_simple_for_command& cmd, const octave_value& bounds) -{ - size_t tc = trip_count (bounds); - if (! tc || ! initialize () || ! enabled ()) - return false; - - jit_info::vmap extra_vars; - extra_vars["#for_bounds0"] = &bounds; - - jit_info *info = cmd.get_info (); - if (! info || ! info->match (extra_vars)) - { - if (tc < static_cast<size_t> (Vjit_startcnt)) - return false; - - delete info; - info = new jit_info (*this, cmd, bounds); - cmd.stash_info (info); - } - - return info->execute (extra_vars); -} - -bool -tree_jit::do_execute (tree_while_command& cmd) -{ - if (! initialize () || ! enabled ()) - return false; - - jit_info *info = cmd.get_info (); - if (! info || ! info->match ()) - { - delete info; - info = new jit_info (*this, cmd); - cmd.stash_info (info); - } - - return info->execute (); -} - -bool -tree_jit::do_execute (octave_user_function& fcn, const octave_value_list& args, - octave_value_list& retval) -{ - if (! initialize () || ! enabled ()) - return false; - - jit_function_info *info = fcn.get_info (); - if (! info || ! info->match (args)) - { - delete info; - info = new jit_function_info (*this, fcn, args); - fcn.stash_info (info); - } - - return info->execute (args, retval); -} - -bool -tree_jit::enabled (void) -{ - // Ideally, we should only disable JIT if there is a breakpoint in the code - // we are about to run. However, we can't figure this out in O(1) time, so - // we conservatively check for the existence of any breakpoints. - return (Vjit_enable && ! bp_table::have_breakpoints () - && ! octave::Vdebug_on_interrupt && ! Vdebug_on_error); -} - -size_t -tree_jit::trip_count (const octave_value& bounds) const -{ - if (bounds.is_range ()) - { - Range rng = bounds.range_value (); - return rng.numel (); - } - - // unsupported type - return 0; -} - -void -tree_jit::optimize (llvm::Function *fn) -{ - if (Vdebug_jit) - llvm::verifyModule (*module); - - module_pass_manager->run (*module); - pass_manager->run (*fn); - - if (Vdebug_jit) - { - std::string error; -#if defined (RAW_FD_OSTREAM_ARG_IS_LLVM_SYS_FS) - llvm::raw_fd_ostream fout ("test.bc", error, - llvm::sys::fs::F_Binary); -#else - llvm::raw_fd_ostream fout ("test.bc", error, - llvm::raw_fd_ostream::F_Binary); -#endif - llvm::WriteBitcodeToFile (module, fout); - } -} - -// -------------------- jit_function_info -------------------- -jit_function_info::jit_function_info (tree_jit& tjit, - octave_user_function& fcn, - const octave_value_list& ov_args) - : argument_types (ov_args.length ()), function (0) -{ - size_t nargs = ov_args.length (); - for (size_t i = 0; i < nargs; ++i) - argument_types[i] = jit_typeinfo::type_of (ov_args(i)); - - jit_function raw_fn; - jit_function wrapper; - - try - { - jit_convert conv (fcn, argument_types); - jit_infer infer (conv.get_factory (), conv.get_blocks (), - conv.get_variable_map ()); - infer.infer (); - - if (Vdebug_jit) - { - jit_block_list& blocks = infer.get_blocks (); - blocks.label (); - std::cout << "-------------------- Compiling function "; - std::cout << "--------------------\n"; - - tree_print_code tpc (std::cout); - tpc.visit_octave_user_function_header (fcn); - tpc.visit_statement_list (*fcn.body ()); - tpc.visit_octave_user_function_trailer (fcn); - blocks.print (std::cout, "octave jit ir"); - } - - jit_factory& factory = conv.get_factory (); - llvm::Module *module = tjit.get_module (); - jit_convert_llvm to_llvm; - raw_fn = to_llvm.convert_function (module, infer.get_blocks (), - factory.constants (), fcn, - argument_types); - - if (Vdebug_jit) - { - std::cout << "-------------------- raw function "; - std::cout << "--------------------\n"; - std::cout << *raw_fn.to_llvm () << std::endl; - llvm::verifyFunction (*raw_fn.to_llvm ()); - } - - std::string wrapper_name = fcn.name () + "_wrapper"; - jit_type *any_t = jit_typeinfo::get_any (); - std::vector<jit_type *> wrapper_args (1, jit_typeinfo::get_any_ptr ()); - wrapper = jit_function (module, jit_convention::internal, wrapper_name, - any_t, wrapper_args); - - llvm::BasicBlock *wrapper_body = wrapper.new_block (); - builder.SetInsertPoint (wrapper_body); - - llvm::Value *wrapper_arg = wrapper.argument (builder, 0); - std::vector<llvm::Value *> raw_args (nargs); - for (size_t i = 0; i < nargs; ++i) - { - llvm::Value *arg; - arg = builder.CreateConstInBoundsGEP1_32 (wrapper_arg, i); - arg = builder.CreateLoad (arg); - - jit_type *arg_type = argument_types[i]; - const jit_function& cast = jit_typeinfo::cast (arg_type, any_t); - raw_args[i] = cast.call (builder, arg); - } - - llvm::Value *result = raw_fn.call (builder, raw_args); - if (raw_fn.result ()) - { - jit_type *raw_result_t = raw_fn.result (); - const jit_function& cast = jit_typeinfo::cast (any_t, raw_result_t); - result = cast.call (builder, result); - } - else - { - llvm::Value *zero = builder.getInt32 (0); - result = builder.CreateBitCast (zero, any_t->to_llvm ()); - } - - wrapper.do_return (builder, result); - - llvm::Function *llvm_function = wrapper.to_llvm (); - tjit.optimize (llvm_function); - - if (Vdebug_jit) - { - std::cout << "-------------------- optimized and wrapped "; - std::cout << "--------------------\n"; - std::cout << *llvm_function << std::endl; - llvm::verifyFunction (*llvm_function); - } - - llvm::ExecutionEngine* engine = tjit.get_engine (); - void *void_fn = engine->getPointerToFunction (llvm_function); - function = reinterpret_cast<jited_function> (void_fn); - } - catch (const jit_fail_exception& e) - { - argument_types.clear (); - - if (Vdebug_jit) - { - if (e.known ()) - std::cout << "jit fail: " << e.what () << std::endl; - } - - Vjit_failcnt++; - - wrapper.erase (); - raw_fn.erase (); - } -} - -bool -jit_function_info::execute (const octave_value_list& ov_args, - octave_value_list& retval) const -{ - if (! function) - return false; - - // FIXME: figure out a way to delete ov_args so we avoid duplicating refcount - size_t nargs = ov_args.length (); - std::vector<octave_base_value *> args (nargs); - for (size_t i = 0; i < nargs; ++i) - { - octave_base_value *obv = ov_args(i).internal_rep (); - obv->grab (); - args[i] = obv; - } - - octave_base_value *ret = function (&args[0]); - if (ret) - retval(0) = octave_value (ret); - - octave_quit (); - - return true; -} - -bool -jit_function_info::match (const octave_value_list& ov_args) const -{ - if (! function) - return true; - - size_t nargs = ov_args.length (); - if (nargs != argument_types.size ()) - return false; - - for (size_t i = 0; i < nargs; ++i) - if (jit_typeinfo::type_of (ov_args(i)) != argument_types[i]) - return false; - - return true; -} - -// -------------------- jit_info -------------------- -jit_info::jit_info (tree_jit& tjit, tree& tee) - : engine (tjit.get_engine ()), function (0), llvm_function (0) -{ - compile (tjit, tee); -} - -jit_info::jit_info (tree_jit& tjit, tree& tee, const octave_value& for_bounds) - : engine (tjit.get_engine ()), function (0), llvm_function (0) -{ - compile (tjit, tee, jit_typeinfo::type_of (for_bounds)); -} - -jit_info::~jit_info (void) -{ - if (llvm_function) - llvm_function->eraseFromParent (); -} - -bool -jit_info::execute (const vmap& extra_vars) const -{ - if (! function) - return false; - - std::vector<octave_base_value *> real_arguments (arguments.size ()); - for (size_t i = 0; i < arguments.size (); ++i) - { - if (arguments[i].second) - { - octave_value current = find (extra_vars, arguments[i].first); - octave_base_value *obv = current.internal_rep (); - obv->grab (); - real_arguments[i] = obv; - } - } - - function (&real_arguments[0]); - - for (size_t i = 0; i < arguments.size (); ++i) - { - const std::string& name = arguments[i].first; - - // do not store for loop bounds temporary - if (name.size () && name[0] != '#') - symbol_table::assign (arguments[i].first, real_arguments[i]); - } - - octave_quit (); - - return true; -} - -bool -jit_info::match (const vmap& extra_vars) const -{ - if (! function) - return true; - - for (size_t i = 0; i < bounds.size (); ++i) - { - const std::string& arg_name = bounds[i].second; - octave_value value = find (extra_vars, arg_name); - jit_type *type = jit_typeinfo::type_of (value); - - // FIXME: Check for a parent relationship - if (type != bounds[i].first) - return false; - } - - return true; -} - -void -jit_info::compile (tree_jit& tjit, tree& tee, jit_type *for_bounds) -{ - try - { - jit_convert conv (tee, for_bounds); - jit_infer infer (conv.get_factory (), conv.get_blocks (), - conv.get_variable_map ()); - - infer.infer (); - - if (Vdebug_jit) - { - jit_block_list& blocks = infer.get_blocks (); - blocks.label (); - std::cout << "-------------------- Compiling tree --------------------\n"; - std::cout << tee.str_print_code () << std::endl; - blocks.print (std::cout, "octave jit ir"); - } - - jit_factory& factory = conv.get_factory (); - jit_convert_llvm to_llvm; - llvm_function = to_llvm.convert_loop (tjit.get_module (), - infer.get_blocks (), - factory.constants ()); - arguments = to_llvm.get_arguments (); - bounds = conv.get_bounds (); - } - catch (const jit_fail_exception& e) - { - if (Vdebug_jit) - { - if (e.known ()) - std::cout << "jit fail: " << e.what () << std::endl; - } - - Vjit_failcnt++; - - } - - if (llvm_function) - { - if (Vdebug_jit) - { - std::cout << "-------------------- llvm ir --------------------"; - std::cout << *llvm_function << std::endl; - llvm::verifyFunction (*llvm_function); - } - - tjit.optimize (llvm_function); - - if (Vdebug_jit) - { - std::cout << "-------------------- optimized llvm ir " - << "--------------------\n"; - std::cout << *llvm_function << std::endl; - } - - void *void_fn = engine->getPointerToFunction (llvm_function); - function = reinterpret_cast<jited_function> (void_fn); - } -} - -octave_value -jit_info::find (const vmap& extra_vars, const std::string& vname) const -{ - vmap::const_iterator iter = extra_vars.find (vname); - return iter == extra_vars.end () ? symbol_table::varval (vname) - : *iter->second; -} - -#endif - -DEFUN (jit_failcnt, args, nargout, - doc: /* -*- texinfo -*- -@deftypefn {} {@var{val} =} jit_failcnt () -@deftypefnx {} {@var{old_val} =} jit_failcnt (@var{new_val}) -@deftypefnx {} {} jit_failcnt (@var{new_val}, "local") -Query or set the internal variable that counts the number of JIT fail -exceptions for Octave's JIT compiler. - -When called from inside a function with the @qcode{"local"} option, the -variable is changed locally for the function and any subroutines it calls. -The original variable value is restored when exiting the function. -@seealso{jit_enable, jit_startcnt, debug_jit} -@end deftypefn */) -{ -#if defined (HAVE_LLVM) - return SET_INTERNAL_VARIABLE (jit_failcnt); -#else - octave_unused_parameter (args); - octave_unused_parameter (nargout); - warn_disabled_feature ("jit_failcnt", "JIT compiling"); - return ovl (); -#endif -} - -DEFUN (debug_jit, args, nargout, - doc: /* -*- texinfo -*- -@deftypefn {} {@var{val} =} debug_jit () -@deftypefnx {} {@var{old_val} =} debug_jit (@var{new_val}) -@deftypefnx {} {} debug_jit (@var{new_val}, "local") -Query or set the internal variable that determines whether -debugging/tracing is enabled for Octave's JIT compiler. - -When called from inside a function with the @qcode{"local"} option, the -variable is changed locally for the function and any subroutines it calls. -The original variable value is restored when exiting the function. -@seealso{jit_enable, jit_startcnt} -@end deftypefn */) -{ -#if defined (HAVE_LLVM) - return SET_INTERNAL_VARIABLE (debug_jit); -#else - octave_unused_parameter (args); - octave_unused_parameter (nargout); - warn_disabled_feature ("debug_jit", "JIT"); - return ovl (); -#endif -} - -DEFUN (jit_enable, args, nargout, - doc: /* -*- texinfo -*- -@deftypefn {} {@var{val} =} jit_enable () -@deftypefnx {} {@var{old_val} =} jit_enable (@var{new_val}) -@deftypefnx {} {} jit_enable (@var{new_val}, "local") -Query or set the internal variable that enables Octave's JIT compiler. - -When called from inside a function with the @qcode{"local"} option, the -variable is changed locally for the function and any subroutines it calls. -The original variable value is restored when exiting the function. -@seealso{jit_startcnt, debug_jit} -@end deftypefn */) -{ -#if defined (HAVE_LLVM) - return SET_INTERNAL_VARIABLE (jit_enable); -#else - octave_unused_parameter (args); - octave_unused_parameter (nargout); - warn_disabled_feature ("jit_enable", "JIT"); - return ovl (); -#endif -} - -DEFUN (jit_startcnt, args, nargout, - doc: /* -*- texinfo -*- -@deftypefn {} {@var{val} =} jit_startcnt () -@deftypefnx {} {@var{old_val} =} jit_startcnt (@var{new_val}) -@deftypefnx {} {} jit_startcnt (@var{new_val}, "local") -Query or set the internal variable that determines whether JIT compilation -will take place for a specific loop. - -Because compilation is a costly operation it does not make sense to employ -JIT when the loop count is low. By default only loops with greater than -1000 iterations will be accelerated. - -When called from inside a function with the @qcode{"local"} option, the -variable is changed locally for the function and any subroutines it calls. -The original variable value is restored when exiting the function. -@seealso{jit_enable, jit_failcnt, debug_jit} -@end deftypefn */) -{ -#if defined (HAVE_LLVM) - return SET_INTERNAL_VARIABLE_WITH_LIMITS (jit_startcnt, 1, - std::numeric_limits<int>::max ()); -#else - octave_unused_parameter (args); - octave_unused_parameter (nargout); - warn_disabled_feature ("jit_enable", "JIT"); - return ovl (); -#endif -}
--- a/libinterp/corefcn/pt-jit.h Tue Jan 31 17:42:29 2017 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,453 +0,0 @@ -/* - -Copyright (C) 2012-2016 Max Brister - -This file is part of Octave. - -Octave is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. - -Octave is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Octave; see the file COPYING. If not, see -<http://www.gnu.org/licenses/>. - -*/ - -// Author: Max Brister <max@2bass.com> - -#if ! defined (octave_pt_jit_h) -#define octave_pt_jit_h 1 - -#include "octave-config.h" - -#if defined (HAVE_LLVM) - -#include "jit-ir.h" -#include "pt-walk.h" -#include "symtab.h" - -class octave_value_list; - -// Convert from the parse tree (AST) to the low level Octave IR. -class -jit_convert : public tree_walker -{ -public: - typedef std::pair<jit_type *, std::string> type_bound; - typedef std::vector<type_bound> type_bound_vector; - typedef std::map<std::string, jit_variable *> variable_map; - - jit_convert (tree &tee, jit_type *for_bounds = 0); - - jit_convert (octave_user_function& fcn, const std::vector<jit_type *>& args); - -#define DECL_ARG(n) const ARG ## n& arg ## n -#define JIT_CREATE_CHECKED(N) \ - template <OCT_MAKE_DECL_LIST (typename, ARG, N)> \ - jit_call *create_checked (OCT_MAKE_LIST (DECL_ARG, N)) \ - { \ - jit_call *ret = factory.create<jit_call> (OCT_MAKE_ARG_LIST (arg, N)); \ - return create_checked_impl (ret); \ - } - - JIT_CREATE_CHECKED (1) - JIT_CREATE_CHECKED (2) - JIT_CREATE_CHECKED (3) - JIT_CREATE_CHECKED (4) - -#undef JIT_CREATE_CHECKED -#undef DECL_ARG - - jit_block_list& get_blocks (void) { return blocks; } - - const type_bound_vector& get_bounds (void) const { return bounds; } - - jit_factory& get_factory (void) { return factory; } - - llvm::Function *get_function (void) const { return function; } - - const variable_map &get_variable_map (void) const { return vmap; } - - void visit_anon_fcn_handle (tree_anon_fcn_handle&); - - void visit_argument_list (tree_argument_list&); - - void visit_binary_expression (tree_binary_expression&); - - void visit_break_command (tree_break_command&); - - void visit_colon_expression (tree_colon_expression&); - - void visit_continue_command (tree_continue_command&); - - void visit_global_command (tree_global_command&); - - void visit_persistent_command (tree_persistent_command&); - - void visit_decl_elt (tree_decl_elt&); - - void visit_decl_init_list (tree_decl_init_list&); - - void visit_simple_for_command (tree_simple_for_command&); - - void visit_complex_for_command (tree_complex_for_command&); - - void visit_octave_user_script (octave_user_script&); - - void visit_octave_user_function (octave_user_function&); - - void visit_octave_user_function_header (octave_user_function&); - - void visit_octave_user_function_trailer (octave_user_function&); - - void visit_function_def (tree_function_def&); - - void visit_identifier (tree_identifier&); - - void visit_if_clause (tree_if_clause&); - - void visit_if_command (tree_if_command&); - - void visit_if_command_list (tree_if_command_list&); - - void visit_index_expression (tree_index_expression&); - - void visit_matrix (tree_matrix&); - - void visit_cell (tree_cell&); - - void visit_multi_assignment (tree_multi_assignment&); - - void visit_no_op_command (tree_no_op_command&); - - void visit_constant (tree_constant&); - - void visit_fcn_handle (tree_fcn_handle&); - - void visit_funcall (tree_funcall&); - - void visit_parameter_list (tree_parameter_list&); - - void visit_postfix_expression (tree_postfix_expression&); - - void visit_prefix_expression (tree_prefix_expression&); - - void visit_return_command (tree_return_command&); - - void visit_return_list (tree_return_list&); - - void visit_simple_assignment (tree_simple_assignment&); - - void visit_statement (tree_statement&); - - void visit_statement_list (tree_statement_list&); - - void visit_switch_case (tree_switch_case&); - - void visit_switch_case_list (tree_switch_case_list&); - - void visit_switch_command (tree_switch_command&); - - void visit_try_catch_command (tree_try_catch_command&); - - void visit_unwind_protect_command (tree_unwind_protect_command&); - - void visit_while_command (tree_while_command&); - - void visit_do_until_command (tree_do_until_command&); -private: - std::vector<std::pair<std::string, bool> > arguments; - type_bound_vector bounds; - - bool converting_function; - - // the scope of the function we are converting, or the current scope - symbol_table::scope_id scope; - - jit_factory factory; - - // used instead of return values from visit_* functions - jit_value *result; - - jit_block *entry_block; - - jit_block *final_block; - - jit_block *block; - - llvm::Function *function; - - jit_block_list blocks; - - std::vector<jit_magic_end::context> end_context; - - size_t iterator_count; - size_t for_bounds_count; - size_t short_count; - - variable_map vmap; - - void initialize (symbol_table::scope_id s); - - jit_call *create_checked_impl (jit_call *ret); - - // get an existing vairable. If the variable does not exist, it will not be - // created - jit_variable *find_variable (const std::string& vname) const; - - // get a variable, create it if it does not exist. The type will default to - // the variable's current type in the symbol table. - jit_variable *get_variable (const std::string& vname); - - // create a variable of the given name and given type. Will also insert an - // extract statement - jit_variable *create_variable (const std::string& vname, jit_type *type, - bool isarg = true); - - // The name of the next for loop iterator. If inc is false, then the - // iterator counter will not be incremented. - std::string next_iterator (bool inc = true) - { return next_name ("#iter", iterator_count, inc); } - - std::string next_for_bounds (bool inc = true) - { return next_name ("#for_bounds", for_bounds_count, inc); } - - std::string next_shortcircut_result (bool inc = true) - { return next_name ("#shortcircut_result", short_count, inc); } - - std::string next_name (const char *prefix, size_t& count, bool inc); - - jit_instruction *resolve (tree_index_expression& exp, - jit_value *extra_arg = 0, bool lhs = false); - - jit_value *do_assign (tree_expression *exp, jit_value *rhs, - bool artificial = false); - - jit_value *do_assign (const std::string& lhs, jit_value *rhs, bool print, - bool artificial = false); - - jit_value *visit (tree *tee) { return visit (*tee); } - - jit_value *visit (tree& tee); - - typedef std::list<jit_block *> block_list; - block_list breaks; - block_list continues; - - void finish_breaks (jit_block *dest, const block_list& lst); -}; - -// Convert from the low level Octave IR to LLVM -class -jit_convert_llvm : public jit_ir_walker -{ -public: - llvm::Function *convert_loop (llvm::Module *module, - const jit_block_list& blocks, - const std::list<jit_value *>& constants); - - jit_function convert_function (llvm::Module *module, - const jit_block_list& blocks, - const std::list<jit_value *>& constants, - octave_user_function& fcn, - const std::vector<jit_type *>& args); - - // arguments to the llvm::Function for loops - const std::vector<std::pair<std::string, bool> >& get_arguments(void) const - { return argument_vec; } - -#define JIT_METH(clname) \ - virtual void visit (jit_ ## clname&); - - JIT_VISIT_IR_CLASSES; - -#undef JIT_METH -private: - // name -> argument index (used for compiling functions) - std::map<std::string, int> argument_index; - - std::vector<std::pair<std::string, bool> > argument_vec; - - // name -> llvm argument (used for compiling loops) - std::map<std::string, llvm::Value *> arguments; - - bool converting_function; - - // only used if we are converting a function - jit_function creating; - - llvm::Function *function; - llvm::BasicBlock *prelude; - - void convert (const jit_block_list& blocks, - const std::list<jit_value *>& constants); - - void finish_phi (jit_phi *phi); - - void visit (jit_value *jvalue) - { - return visit (*jvalue); - } - - void visit (jit_value &jvalue) - { - jvalue.accept (*this); - } -}; - -// type inference and SSA construction on the low level Octave IR -class -jit_infer -{ -public: - typedef jit_convert::variable_map variable_map; - - jit_infer (jit_factory& afactory, jit_block_list& ablocks, - const variable_map& avmap); - - jit_block_list& get_blocks (void) const { return blocks; } - - jit_factory& get_factory (void) const { return factory; } - - void infer (void); -private: - jit_block_list& blocks; - jit_factory& factory; - const variable_map& vmap; - std::list<jit_instruction *> worklist; - - void append_users (jit_value *v); - - void append_users_term (jit_terminator *term); - - void construct_ssa (void); - - void do_construct_ssa (jit_block& block, size_t avisit_count); - - jit_block& entry_block (void) { return *blocks.front (); } - - jit_block& final_block (void) { return *blocks.back (); } - - void place_releases (void); - - void push_worklist (jit_instruction *instr); - - void remove_dead (); - - void release_dead_phi (jit_block& ablock); - - void release_temp (jit_block& ablock, std::set<jit_value *>& temp); - - void simplify_phi (void); - - void simplify_phi (jit_phi& phi); -}; - -class -tree_jit -{ -public: - ~tree_jit (void); - - static bool execute (tree_simple_for_command& cmd, - const octave_value& bounds); - - static bool execute (tree_while_command& cmd); - - static bool execute (octave_user_function& fcn, const octave_value_list& args, - octave_value_list& retval); - - llvm::ExecutionEngine *get_engine (void) const { return engine; } - - llvm::Module *get_module (void) const { return module; } - - void optimize (llvm::Function *fn); -private: - tree_jit (void); - - static tree_jit& instance (void); - - bool initialize (void); - - bool do_execute (tree_simple_for_command& cmd, const octave_value& bounds); - - bool do_execute (tree_while_command& cmd); - - bool do_execute (octave_user_function& fcn, const octave_value_list& args, - octave_value_list& retval); - - bool enabled (void); - - size_t trip_count (const octave_value& bounds) const; - - llvm::Module *module; -#if defined (LEGACY_PASSMANAGER) - llvm::legacy::PassManager *module_pass_manager; - llvm::legacy::FunctionPassManager *pass_manager; -#else - llvm::PassManager *module_pass_manager; - llvm::FunctionPassManager *pass_manager; -#endif - llvm::ExecutionEngine *engine; -}; - -class -jit_function_info -{ -public: - jit_function_info (tree_jit& tjit, octave_user_function& fcn, - const octave_value_list& ov_args); - - bool execute (const octave_value_list& ov_args, - octave_value_list& retval) const; - - bool match (const octave_value_list& ov_args) const; -private: - typedef octave_base_value *(*jited_function)(octave_base_value**); - - std::vector<jit_type *> argument_types; - jited_function function; -}; - -class -jit_info -{ -public: - // we use a pointer here so we don't have to include ov.h - typedef std::map<std::string, const octave_value *> vmap; - - jit_info (tree_jit& tjit, tree& tee); - - jit_info (tree_jit& tjit, tree& tee, const octave_value& for_bounds); - - ~jit_info (void); - - bool execute (const vmap& extra_vars = vmap ()) const; - - bool match (const vmap& extra_vars = vmap ()) const; -private: - typedef jit_convert::type_bound type_bound; - typedef jit_convert::type_bound_vector type_bound_vector; - typedef void (*jited_function)(octave_base_value**); - - void compile (tree_jit& tjit, tree& tee, jit_type *for_bounds = 0); - - octave_value find (const vmap& extra_vars, const std::string& vname) const; - - llvm::ExecutionEngine *engine; - jited_function function; - llvm::Function *llvm_function; - - std::vector<std::pair<std::string, bool> > arguments; - type_bound_vector bounds; -}; - -#endif -#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-ir.cc Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,845 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +// defines required by llvm +#define __STDC_LIMIT_MACROS +#define __STDC_CONSTANT_MACROS + +#if defined (HAVE_CONFIG_H) +# include "config.h" +#endif + +#if defined (HAVE_LLVM) + +#include "jit-ir.h" + +#if defined (HAVE_LLVM_IR_FUNCTION_H) +# include <llvm/IR/BasicBlock.h> +# include <llvm/IR/Instructions.h> +#else +# include <llvm/BasicBlock.h> +# include <llvm/Instructions.h> +#endif + +#include "error.h" + +// -------------------- jit_factory -------------------- +jit_factory::~jit_factory (void) +{ + for (value_list::iterator iter = all_values.begin (); + iter != all_values.end (); ++iter) + delete *iter; +} + +void +jit_factory::track_value (jit_value *value) +{ + if (value->type ()) + mconstants.push_back (value); + all_values.push_back (value); +} + +// -------------------- jit_block_list -------------------- +void +jit_block_list::insert_after (iterator iter, jit_block *ablock) +{ + ++iter; + insert_before (iter, ablock); +} + +void +jit_block_list::insert_after (jit_block *loc, jit_block *ablock) +{ + insert_after (loc->location (), ablock); +} + +void +jit_block_list::insert_before (iterator iter, jit_block *ablock) +{ + iter = mlist.insert (iter, ablock); + ablock->stash_location (iter); +} + +void +jit_block_list::insert_before (jit_block *loc, jit_block *ablock) +{ + insert_before (loc->location (), ablock); +} + +void +jit_block_list::label (void) +{ + if (mlist.size ()) + { + jit_block *block = mlist.back (); + block->label (); + } +} + +std::ostream& +jit_block_list::print (std::ostream& os, const std::string& header) const +{ + os << "-------------------- " << header << " --------------------\n"; + return os << *this; +} + +std::ostream& +jit_block_list::print_dom (std::ostream& os) const +{ + os << "-------------------- dom info --------------------\n"; + for (const_iterator iter = begin (); iter != end (); ++iter) + { + assert (*iter); + (*iter)->print_dom (os); + } + os << std::endl; + + return os; +} + +void +jit_block_list::push_back (jit_block *b) +{ + mlist.push_back (b); + iterator iter = mlist.end (); + b->stash_location (--iter); +} + +std::ostream& +operator<<(std::ostream& os, const jit_block_list& blocks) +{ + for (jit_block_list::const_iterator iter = blocks.begin (); + iter != blocks.end (); ++iter) + { + assert (*iter); + (*iter)->print (os, 0); + } + return os << std::endl; +} + +// -------------------- jit_use -------------------- +jit_block * +jit_use::user_parent (void) const +{ + return muser->parent (); +} + +// -------------------- jit_value -------------------- +jit_value::~jit_value (void) +{ } + +jit_block * +jit_value::first_use_block (void) +{ + jit_use *use = first_use (); + while (use) + { + if (! isa<jit_error_check> (use->user ())) + return use->user_parent (); + + use = use->next (); + } + + return 0; +} + +void +jit_value::replace_with (jit_value *value) +{ + while (first_use ()) + { + jit_instruction *user = first_use ()->user (); + size_t idx = first_use ()->index (); + user->stash_argument (idx, value); + } +} + +#define JIT_METH(clname) \ + void \ + jit_ ## clname::accept (jit_ir_walker& walker) \ + { \ + walker.visit (*this); \ + } + +JIT_VISIT_IR_NOTEMPLATE +#undef JIT_METH + +std::ostream& +operator<< (std::ostream& os, const jit_value& value) +{ + return value.short_print (os); +} + +std::ostream& +jit_print (std::ostream& os, jit_value *avalue) +{ + if (avalue) + return avalue->print (os); + return os << "NULL"; +} + +// -------------------- jit_instruction -------------------- +void +jit_instruction::remove (void) +{ + if (mparent) + mparent->remove (mlocation); + resize_arguments (0); +} + +llvm::BasicBlock * +jit_instruction::parent_llvm (void) const +{ + return mparent->to_llvm (); +} + +std::ostream& +jit_instruction::short_print (std::ostream& os) const +{ + if (type ()) + jit_print (os, type ()) << ": "; + return os << "#" << mid; +} + +void +jit_instruction::do_construct_ssa (size_t start, size_t end) +{ + for (size_t i = start; i < end; ++i) + { + jit_value *arg = argument (i); + jit_variable *var = dynamic_cast<jit_variable *> (arg); + if (var && var->has_top ()) + stash_argument (i, var->top ()); + } +} + +// -------------------- jit_block -------------------- +void +jit_block::replace_with (jit_value *value) +{ + assert (isa<jit_block> (value)); + jit_block *block = static_cast<jit_block *> (value); + + jit_value::replace_with (block); + + while (ILIST_T::first_use ()) + { + jit_phi_incomming *incomming = ILIST_T::first_use (); + incomming->stash_value (block); + } +} + +void +jit_block::replace_in_phi (jit_block *ablock, jit_block *with) +{ + jit_phi_incomming *node = ILIST_T::first_use (); + while (node) + { + jit_phi_incomming *prev = node; + node = node->next (); + + if (prev->user_parent () == ablock) + prev->stash_value (with); + } +} + +jit_block * +jit_block::maybe_merge () +{ + if (successor_count () == 1 && successor (0) != this + && (successor (0)->use_count () == 1 || instructions.size () == 1)) + { + jit_block *to_merge = successor (0); + merge (*to_merge); + return to_merge; + } + + return 0; +} + +void +jit_block::merge (jit_block& block) +{ + // the merge block will contain a new terminator + jit_terminator *old_term = terminator (); + if (old_term) + old_term->remove (); + + bool was_empty = end () == begin (); + iterator merge_begin = end (); + if (! was_empty) + --merge_begin; + + instructions.splice (end (), block.instructions); + if (was_empty) + merge_begin = begin (); + else + ++merge_begin; + + // now merge_begin points to the start of the new instructions, we must + // update their parent information + for (iterator iter = merge_begin; iter != end (); ++iter) + { + jit_instruction *instr = *iter; + instr->stash_parent (this, iter); + } + + block.replace_with (this); +} + +jit_instruction * +jit_block::prepend (jit_instruction *instr) +{ + instructions.push_front (instr); + instr->stash_parent (this, instructions.begin ()); + return instr; +} + +jit_instruction * +jit_block::prepend_after_phi (jit_instruction *instr) +{ + // FIXME: Make this O(1) + for (iterator iter = begin (); iter != end (); ++iter) + { + jit_instruction *temp = *iter; + if (! isa<jit_phi> (temp)) + { + insert_before (iter, instr); + return instr; + } + } + + return append (instr); +} + +void +jit_block::internal_append (jit_instruction *instr) +{ + instructions.push_back (instr); + instr->stash_parent (this, --instructions.end ()); +} + +jit_instruction * +jit_block::insert_before (iterator loc, jit_instruction *instr) +{ + iterator iloc = instructions.insert (loc, instr); + instr->stash_parent (this, iloc); + return instr; +} + +jit_instruction * +jit_block::insert_after (iterator loc, jit_instruction *instr) +{ + ++loc; + iterator iloc = instructions.insert (loc, instr); + instr->stash_parent (this, iloc); + return instr; +} + +jit_terminator * +jit_block::terminator (void) const +{ + assert (this); + if (instructions.empty ()) + return 0; + + jit_instruction *last = instructions.back (); + return dynamic_cast<jit_terminator *> (last); +} + +bool +jit_block::branch_alive (jit_block *asucc) const +{ + return terminator ()->alive (asucc); +} + +jit_block * +jit_block::successor (size_t i) const +{ + jit_terminator *term = terminator (); + return term->successor (i); +} + +size_t +jit_block::successor_count (void) const +{ + jit_terminator *term = terminator (); + return term ? term->successor_count () : 0; +} + +llvm::BasicBlock * +jit_block::to_llvm (void) const +{ + return llvm::cast<llvm::BasicBlock> (llvm_value); +} + +std::ostream& +jit_block::print_dom (std::ostream& os) const +{ + short_print (os); + os << ":\n"; + os << " mid: " << mid << std::endl; + os << " predecessors: "; + for (jit_use *use = first_use (); use; use = use->next ()) + os << *use->user_parent () << " "; + os << std::endl; + + os << " successors: "; + for (size_t i = 0; i < successor_count (); ++i) + os << *successor (i) << " "; + os << std::endl; + + os << " idom: "; + if (idom) + os << *idom; + else + os << "NULL"; + os << std::endl; + os << " df: "; + for (df_iterator iter = df_begin (); iter != df_end (); ++iter) + os << **iter << " "; + os << std::endl; + + os << " dom_succ: "; + for (size_t i = 0; i < dom_succ.size (); ++i) + os << *dom_succ[i] << " "; + + return os << std::endl; +} + +void +jit_block::compute_df (size_t avisit_count) +{ + if (visited (avisit_count)) + return; + + if (use_count () >= 2) + { + for (jit_use *use = first_use (); use; use = use->next ()) + { + jit_block *runner = use->user_parent (); + while (runner != idom) + { + runner->mdf.insert (this); + runner = runner->idom; + } + } + } + + for (size_t i = 0; i < successor_count (); ++i) + successor (i)->compute_df (avisit_count); +} + +bool +jit_block::update_idom (size_t avisit_count) +{ + if (visited (avisit_count) || ! use_count ()) + return false; + + bool changed = false; + for (jit_use *use = first_use (); use; use = use->next ()) + { + jit_block *pred = use->user_parent (); + changed = pred->update_idom (avisit_count) || changed; + } + + jit_use *use = first_use (); + jit_block *new_idom = use->user_parent (); + use = use->next (); + + for (; use; use = use->next ()) + { + jit_block *pred = use->user_parent (); + jit_block *pidom = pred->idom; + if (pidom) + new_idom = idom_intersect (pidom, new_idom); + } + + if (idom != new_idom) + { + idom = new_idom; + return true; + } + + return changed; +} + +void +jit_block::label (size_t avisit_count, size_t& number) +{ + if (visited (avisit_count)) + return; + + for (jit_use *use = first_use (); use; use = use->next ()) + { + jit_block *pred = use->user_parent (); + pred->label (avisit_count, number); + } + + mid = number++; +} + +void +jit_block::pop_all (void) +{ + for (iterator iter = begin (); iter != end (); ++iter) + { + jit_instruction *instr = *iter; + instr->pop_variable (); + } +} + +std::ostream& +jit_block::print (std::ostream& os, size_t indent) const +{ + print_indent (os, indent); + short_print (os) << ": %pred = "; + for (jit_use *use = first_use (); use; use = use->next ()) + { + jit_block *pred = use->user_parent (); + os << *pred; + if (use->next ()) + os << ", "; + } + os << std::endl; + + for (const_iterator iter = begin (); iter != end (); ++iter) + { + jit_instruction *instr = *iter; + instr->print (os, indent + 1) << std::endl; + } + return os; +} + +jit_block * +jit_block::maybe_split (jit_factory& factory, jit_block_list& blocks, + jit_block *asuccessor) +{ + if (successor_count () > 1) + { + jit_terminator *term = terminator (); + size_t idx = term->successor_index (asuccessor); + jit_block *split = factory.create<jit_block> ("phi_split", mvisit_count); + + // place after this to ensure define before use in the blocks list + blocks.insert_after (this, split); + + term->stash_argument (idx, split); + jit_branch *br = split->append (factory.create<jit_branch> (asuccessor)); + replace_in_phi (asuccessor, split); + + if (alive ()) + { + split->mark_alive (); + br->infer (); + } + + return split; + } + + return this; +} + +void +jit_block::create_dom_tree (size_t avisit_count) +{ + if (visited (avisit_count)) + return; + + if (idom != this) + idom->dom_succ.push_back (this); + + for (size_t i = 0; i < successor_count (); ++i) + successor (i)->create_dom_tree (avisit_count); +} + +jit_block * +jit_block::idom_intersect (jit_block *i, jit_block *j) +{ + while (i && j && i != j) + { + while (i && i->id () > j->id ()) + i = i->idom; + + while (i && j && j->id () > i->id ()) + j = j->idom; + } + + return i ? i : j; +} + +// -------------------- jit_phi_incomming -------------------- + +jit_block * +jit_phi_incomming::user_parent (void) const +{ return muser->parent (); } + +// -------------------- jit_phi -------------------- +bool +jit_phi::prune (void) +{ + jit_block *p = parent (); + size_t new_idx = 0; + jit_value *unique = argument (1); + + for (size_t i = 0; i < argument_count (); ++i) + { + jit_block *inc = incomming (i); + if (inc->branch_alive (p)) + { + if (unique != argument (i)) + unique = 0; + + if (new_idx != i) + { + stash_argument (new_idx, argument (i)); + mincomming[new_idx].stash_value (inc); + } + + ++new_idx; + } + } + + if (new_idx != argument_count ()) + { + resize_arguments (new_idx); + mincomming.resize (new_idx); + } + + assert (argument_count () > 0); + if (unique) + { + replace_with (unique); + return true; + } + + return false; +} + +bool +jit_phi::infer (void) +{ + jit_block *p = parent (); + if (! p->alive ()) + return false; + + jit_type *infered = 0; + for (size_t i = 0; i < argument_count (); ++i) + { + jit_block *inc = incomming (i); + if (inc->branch_alive (p)) + infered = jit_typeinfo::join (infered, argument_type (i)); + } + + if (infered != type ()) + { + stash_type (infered); + return true; + } + + return false; +} + +llvm::PHINode * +jit_phi::to_llvm (void) const +{ + return llvm::cast<llvm::PHINode> (jit_value::to_llvm ()); +} + +// -------------------- jit_terminator -------------------- +size_t +jit_terminator::successor_index (const jit_block *asuccessor) const +{ + size_t scount = successor_count (); + for (size_t i = 0; i < scount; ++i) + if (successor (i) == asuccessor) + return i; + + panic_impossible (); +} + +bool +jit_terminator::infer (void) +{ + if (! parent ()->alive ()) + return false; + + bool changed = false; + for (size_t i = 0; i < malive.size (); ++i) + if (! malive[i] && check_alive (i)) + { + changed = true; + malive[i] = true; + successor (i)->mark_alive (); + } + + return changed; +} + +llvm::TerminatorInst * +jit_terminator::to_llvm (void) const +{ + return llvm::cast<llvm::TerminatorInst> (jit_value::to_llvm ()); +} + +// -------------------- jit_call -------------------- +bool +jit_call::needs_release (void) const +{ + if (type () && jit_typeinfo::get_release (type ()).valid ()) + { + for (jit_use *use = first_use (); use; use = use->next ()) + { + jit_assign *assign = dynamic_cast<jit_assign *> (use->user ()); + if (assign && assign->artificial ()) + return false; + } + + return true; + } + return false; +} + +bool +jit_call::infer (void) +{ + // FIXME: explain algorithm + for (size_t i = 0; i < argument_count (); ++i) + { + already_infered[i] = argument_type (i); + if (! already_infered[i]) + return false; + } + + jit_type *infered = moperation.result (already_infered); + if (! infered && use_count ()) + { + std::stringstream ss; + ss << "Missing overload in type inference for "; + print (ss, 0); + throw jit_fail_exception (ss.str ()); + } + + if (infered != type ()) + { + stash_type (infered); + return true; + } + + return false; +} + +// -------------------- jit_error_check -------------------- +std::string +jit_error_check::variable_to_string (variable v) +{ + switch (v) + { + case var_error_state: + return "error_state"; + case var_interrupt: + return "interrupt"; + default: + panic_impossible (); + } +} + +std::ostream& +jit_error_check::print (std::ostream& os, size_t indent) const +{ + print_indent (os, indent) << "error_check " << variable_to_string (mvariable) + << ", "; + + if (has_check_for ()) + os << "<for> " << *check_for () << ", "; + print_successor (os << "<normal> ", 1) << ", "; + return print_successor (os << "<error> ", 0); +} + +// -------------------- jit_magic_end -------------------- +jit_magic_end::context::context (jit_factory& factory, jit_value *avalue, + size_t aindex, size_t acount) + : value (avalue), index (factory.create<jit_const_index> (aindex)), + count (factory.create<jit_const_index> (acount)) +{ } + +jit_magic_end::jit_magic_end (const std::vector<context>& full_context) + : contexts (full_context) +{ + resize_arguments (contexts.size ()); + + size_t i; + std::vector<context>::const_iterator iter; + for (iter = contexts.begin (), i = 0; iter != contexts.end (); ++iter, ++i) + stash_argument (i, iter->value); +} + +jit_magic_end::context +jit_magic_end::resolve_context (void) const +{ + size_t idx; + for (idx = 0; idx < contexts.size (); ++idx) + { + jit_type *ctx_type = contexts[idx].value->type (); + if (! ctx_type || ctx_type->skip_paren ()) + break; + } + + if (idx >= contexts.size ()) + idx = 0; + + context ret = contexts[idx]; + ret.value = argument (idx); + return ret; +} + +bool +jit_magic_end::infer (void) +{ + jit_type *new_type = overload ().result (); + if (new_type != type ()) + { + stash_type (new_type); + return true; + } + + return false; +} + +std::ostream& +jit_magic_end::print (std::ostream& os, size_t indent) const +{ + context ctx = resolve_context (); + short_print (print_indent (os, indent)) << " (" << *ctx.value << ", "; + return os << *ctx.index << ", " << *ctx.count << ")"; +} + +const jit_function& +jit_magic_end::overload () const +{ + const context& ctx = resolve_context (); + return jit_typeinfo::end (ctx.value, ctx.index, ctx.count); +} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-ir.h Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,1444 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +#if ! defined (octave_jit_ir_h) +#define octave_jit_ir_h 1 + +#include "octave-config.h" + +#if defined (HAVE_LLVM) + +#include <list> +#include <stack> +#include <set> + +#include "jit-typeinfo.h" + +// The low level octave jit ir +// this ir is close to llvm, but contains information for doing type inference. +// We convert the octave parse tree to this IR directly. + +#define JIT_VISIT_IR_NOTEMPLATE \ + JIT_METH(block); \ + JIT_METH(branch); \ + JIT_METH(cond_branch); \ + JIT_METH(call); \ + JIT_METH(extract_argument); \ + JIT_METH(store_argument); \ + JIT_METH(return); \ + JIT_METH(phi); \ + JIT_METH(variable); \ + JIT_METH(error_check); \ + JIT_METH(assign) \ + JIT_METH(argument) \ + JIT_METH(magic_end) + +#define JIT_VISIT_IR_CONST \ + JIT_METH(const_bool); \ + JIT_METH(const_scalar); \ + JIT_METH(const_complex); \ + JIT_METH(const_index); \ + JIT_METH(const_string); \ + JIT_METH(const_range) + +#define JIT_VISIT_IR_CLASSES \ + JIT_VISIT_IR_NOTEMPLATE \ + JIT_VISIT_IR_CONST + +// forward declare all ir classes +#define JIT_METH(cname) \ + class jit_ ## cname; + +JIT_VISIT_IR_NOTEMPLATE + +#undef JIT_METH + +// ABCs which aren't included in JIT_VISIT_IR_ALL +class jit_instruction; +class jit_terminator; + +template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T = T, + bool QUOTE=false> +class jit_const; + +typedef jit_const<bool, jit_typeinfo::get_bool> jit_const_bool; +typedef jit_const<double, jit_typeinfo::get_scalar> jit_const_scalar; +typedef jit_const<Complex, jit_typeinfo::get_complex> jit_const_complex; +typedef jit_const<octave_idx_type, jit_typeinfo::get_index> jit_const_index; + +typedef jit_const<std::string, jit_typeinfo::get_string, const std::string&, + true> jit_const_string; +typedef jit_const<jit_range, jit_typeinfo::get_range, const jit_range&> +jit_const_range; + +class jit_ir_walker; +class jit_use; + +// Creates and tracks memory for jit_value and subclasses. +// Memory managment is simple, all values that are created live as long as the +// factory. +class +jit_factory +{ + typedef std::list<jit_value *> value_list; + +public: + + ~jit_factory (void); + + const value_list& constants (void) const { return mconstants; } + + template <typename T> + T *create (void) + { + T *ret = new T (); + track_value (ret); + return ret; + } + +#define DECL_ARG(n) const ARG ## n& arg ## n + +#define JIT_CREATE(N) \ + template <typename T, OCT_MAKE_DECL_LIST (typename, ARG, N)> \ + T *create (OCT_MAKE_LIST (DECL_ARG, N)) \ + { \ + T *ret = new T (OCT_MAKE_ARG_LIST (arg, N)); \ + track_value (ret); \ + return ret; \ + } + + JIT_CREATE (1) + JIT_CREATE (2) + JIT_CREATE (3) + JIT_CREATE (4) + +#undef JIT_CREATE +#undef DECL_ARG + +private: + + void track_value (jit_value *v); + + value_list all_values; + + value_list mconstants; +}; + +// A list of basic blocks (jit_block) which form some body of code. +// +// We do not directly inherit from std::list because we need to update the +// blocks stashed location in push_back and insert. +class +jit_block_list +{ +public: + typedef std::list<jit_block *>::iterator iterator; + typedef std::list<jit_block *>::const_iterator const_iterator; + + jit_block *back (void) const { return mlist.back (); } + + iterator begin (void) { return mlist.begin (); } + + const_iterator begin (void) const { return mlist.begin (); } + + iterator end (void) { return mlist.end (); } + + const_iterator end (void) const { return mlist.end (); } + + iterator erase (iterator iter) { return mlist.erase (iter); } + + jit_block *front (void) const { return mlist.front (); } + + void insert_after (iterator iter, jit_block *ablock); + + void insert_after (jit_block *loc, jit_block *ablock); + + void insert_before (iterator iter, jit_block *ablock); + + void insert_before (jit_block *loc, jit_block *ablock); + + void label (void); + + std::ostream& print (std::ostream& os, const std::string& header) const; + + std::ostream& print_dom (std::ostream& os) const; + + void push_back (jit_block *b); +private: + std::list<jit_block *> mlist; +}; + +std::ostream& operator<<(std::ostream& os, const jit_block_list& blocks); + +class +jit_value : public jit_internal_list<jit_value, jit_use> +{ +public: + jit_value (void) : llvm_value (0), ty (0), mlast_use (0), + min_worklist (false) { } + + virtual ~jit_value (void); + + bool in_worklist (void) const + { + return min_worklist; + } + + void stash_in_worklist (bool ain_worklist) + { + min_worklist = ain_worklist; + } + + // The block of the first use which is not a jit_error_check + // So this is not necessarily first_use ()->parent (). + jit_block *first_use_block (void); + + // replace all uses with + virtual void replace_with (jit_value *value); + + jit_type *type (void) const { return ty; } + + llvm::Type *type_llvm (void) const + { + return ty ? ty->to_llvm () : 0; + } + + const std::string& type_name (void) const + { + return ty->name (); + } + + void stash_type (jit_type *new_ty) { ty = new_ty; } + + std::string print_string (void) + { + std::stringstream ss; + print (ss); + return ss.str (); + } + + jit_instruction *last_use (void) const { return mlast_use; } + + void stash_last_use (jit_instruction *alast_use) + { + mlast_use = alast_use; + } + + virtual bool needs_release (void) const { return false; } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const = 0; + + virtual std::ostream& short_print (std::ostream& os) const + { return print (os); } + + virtual void accept (jit_ir_walker& walker) = 0; + + bool has_llvm (void) const + { + return llvm_value; + } + + llvm::Value *to_llvm (void) const + { + assert (llvm_value); + return llvm_value; + } + + void stash_llvm (llvm::Value *compiled) + { + llvm_value = compiled; + } + +protected: + std::ostream& print_indent (std::ostream& os, size_t indent = 0) const + { + for (size_t i = 0; i < indent * 8; ++i) + os << " "; + return os; + } + + llvm::Value *llvm_value; +private: + jit_type *ty; + jit_instruction *mlast_use; + bool min_worklist; +}; + +std::ostream& operator<< (std::ostream& os, const jit_value& value); +std::ostream& jit_print (std::ostream& os, jit_value *avalue); + +class +jit_use : public jit_internal_node<jit_value, jit_use> +{ +public: + // some compilers don't allow us to use jit_internal_node without template + // paremeters + typedef jit_internal_node<jit_value, jit_use> PARENT_T; + + jit_use (void) : muser (0), mindex (0) { } + + // we should really have a move operator, but not until c++11 :( + jit_use (const jit_use& use) : muser (0), mindex (0) + { + *this = use; + } + + jit_use& operator= (const jit_use& use) + { + stash_value (use.value (), use.user (), use.index ()); + return *this; + } + + size_t index (void) const { return mindex; } + + jit_instruction *user (void) const { return muser; } + + jit_block *user_parent (void) const; + + std::list<jit_block *> user_parent_location (void) const; + + void stash_value (jit_value *avalue, jit_instruction *auser = 0, + size_t aindex = -1) + { + PARENT_T::stash_value (avalue); + mindex = aindex; + muser = auser; + } +private: + jit_instruction *muser; + size_t mindex; +}; + +class +jit_instruction : public jit_value +{ +public: + // FIXME: this code could be so much pretier with varadic templates... + jit_instruction (void) : mid (next_id ()), mparent (0) + { } + + jit_instruction (size_t nargs) : mid (next_id ()), mparent (0) + { + already_infered.reserve (nargs); + marguments.reserve (nargs); + } + +#define STASH_ARG(i) stash_argument (i, arg ## i); + +#define JIT_INSTRUCTION_CTOR(N) \ + jit_instruction (OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ + : already_infered (N), marguments (N), mid (next_id ()), mparent (0) \ + { \ + OCT_ITERATE_MACRO (STASH_ARG, N); \ + } + + JIT_INSTRUCTION_CTOR(1) + JIT_INSTRUCTION_CTOR(2) + JIT_INSTRUCTION_CTOR(3) + JIT_INSTRUCTION_CTOR(4) + +#undef STASH_ARG +#undef JIT_INSTRUCTION_CTOR + + jit_instruction (const std::vector<jit_value *>& aarguments) + : already_infered (aarguments.size ()), marguments (aarguments.size ()), + mid (next_id ()), mparent (0) + { + for (size_t i = 0; i < aarguments.size (); ++i) + stash_argument (i, aarguments[i]); + } + + static void reset_ids (void) + { + next_id (true); + } + + jit_value *argument (size_t i) const + { + return marguments[i].value (); + } + + llvm::Value *argument_llvm (size_t i) const + { + assert (argument (i)); + return argument (i)->to_llvm (); + } + + jit_type *argument_type (size_t i) const + { + return argument (i)->type (); + } + + llvm::Type *argument_type_llvm (size_t i) const + { + assert (argument (i)); + return argument_type (i)->to_llvm (); + } + + std::ostream& print_argument (std::ostream& os, size_t i) const + { + if (argument (i)) + return argument (i)->short_print (os); + else + return os << "NULL"; + } + + void stash_argument (size_t i, jit_value *arg) + { + marguments[i].stash_value (arg, this, i); + } + + void push_argument (jit_value *arg) + { + marguments.push_back (jit_use ()); + stash_argument (marguments.size () - 1, arg); + already_infered.push_back (0); + } + + size_t argument_count (void) const + { + return marguments.size (); + } + + void resize_arguments (size_t acount, jit_value *adefault = 0) + { + size_t old = marguments.size (); + marguments.resize (acount); + already_infered.resize (acount); + + if (adefault) + for (size_t i = old; i < acount; ++i) + stash_argument (i, adefault); + } + + const std::vector<jit_use>& arguments (void) const { return marguments; } + + // argument types which have been infered already + const std::vector<jit_type *>& argument_types (void) const + { return already_infered; } + + virtual void push_variable (void) { } + + virtual void pop_variable (void) { } + + virtual void construct_ssa (void) + { + do_construct_ssa (0, argument_count ()); + } + + virtual bool infer (void) { return false; } + + void remove (void); + + virtual std::ostream& short_print (std::ostream& os) const; + + jit_block *parent (void) const { return mparent; } + + std::list<jit_instruction *>::iterator location (void) const + { + return mlocation; + } + + llvm::BasicBlock *parent_llvm (void) const; + + void stash_parent (jit_block *aparent, + std::list<jit_instruction *>::iterator alocation) + { + mparent = aparent; + mlocation = alocation; + } + + size_t id (void) const { return mid; } +protected: + + // Do SSA replacement on arguments in [start, end) + void do_construct_ssa (size_t start, size_t end); + + std::vector<jit_type *> already_infered; +private: + static size_t next_id (bool reset = false) + { + static size_t ret = 0; + if (reset) + return ret = 0; + + return ret++; + } + + std::vector<jit_use> marguments; + + size_t mid; + jit_block *mparent; + std::list<jit_instruction *>::iterator mlocation; +}; + +// defnie accept methods for subclasses +#define JIT_VALUE_ACCEPT \ + virtual void accept (jit_ir_walker& walker); + +// for use as a dummy argument during conversion to LLVM +class +jit_argument : public jit_value +{ +public: + jit_argument (jit_type *atype, llvm::Value *avalue) + { + stash_type (atype); + stash_llvm (avalue); + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent); + return jit_print (os, type ()) << ": DUMMY"; + } + + JIT_VALUE_ACCEPT; +}; + +template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T, bool QUOTE> +class +jit_const : public jit_value +{ +public: + typedef PASS_T pass_t; + + jit_const (PASS_T avalue) : mvalue (avalue) + { + stash_type (EXTRACT_T ()); + } + + PASS_T value (void) const { return mvalue; } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent); + jit_print (os, type ()) << ": "; + if (QUOTE) + os << "\""; + os << mvalue; + if (QUOTE) + os << "\""; + return os; + } + + JIT_VALUE_ACCEPT; +private: + T mvalue; +}; + +class jit_phi_incomming; + +class +jit_block : public jit_value, public jit_internal_list<jit_block, + jit_phi_incomming> +{ + typedef jit_internal_list<jit_block, jit_phi_incomming> ILIST_T; +public: + typedef std::list<jit_instruction *> instruction_list; + typedef instruction_list::iterator iterator; + typedef instruction_list::const_iterator const_iterator; + + typedef std::set<jit_block *> df_set; + typedef df_set::const_iterator df_iterator; + + static const size_t NO_ID = static_cast<size_t> (-1); + + jit_block (const std::string& aname, size_t avisit_count = 0) + : mvisit_count (avisit_count), mid (NO_ID), idom (0), mname (aname), + malive (false) + { } + + virtual void replace_with (jit_value *value); + + void replace_in_phi (jit_block *ablock, jit_block *with); + + // we have a new internal list, but we want to stay compatible with jit_value + jit_use *first_use (void) const { return jit_value::first_use (); } + + size_t use_count (void) const { return jit_value::use_count (); } + + // if a block is alive, then it might be visited during execution + bool alive (void) const { return malive; } + + void mark_alive (void) { malive = true; } + + // If we can merge with a successor, do so and return the now empty block + jit_block *maybe_merge (); + + // merge another block into this block, leaving the merge block empty + void merge (jit_block& merge); + + const std::string& name (void) const { return mname; } + + jit_instruction *prepend (jit_instruction *instr); + + jit_instruction *prepend_after_phi (jit_instruction *instr); + + template <typename T> + T *append (T *instr) + { + internal_append (instr); + return instr; + } + + jit_instruction *insert_before (iterator loc, jit_instruction *instr); + + jit_instruction *insert_before (jit_instruction *loc, jit_instruction *instr) + { + return insert_before (loc->location (), instr); + } + + jit_instruction *insert_after (iterator loc, jit_instruction *instr); + + jit_instruction *insert_after (jit_instruction *loc, jit_instruction *instr) + { + return insert_after (loc->location (), instr); + } + + iterator remove (iterator iter) + { + jit_instruction *instr = *iter; + iter = instructions.erase (iter); + instr->stash_parent (0, instructions.end ()); + return iter; + } + + jit_terminator *terminator (void) const; + + // is the jump from pred alive? + bool branch_alive (jit_block *asucc) const; + + jit_block *successor (size_t i) const; + + size_t successor_count (void) const; + + iterator begin (void) { return instructions.begin (); } + + const_iterator begin (void) const { return instructions.begin (); } + + iterator end (void) { return instructions.end (); } + + const_iterator end (void) const { return instructions.end (); } + + iterator phi_begin (void); + + iterator phi_end (void); + + iterator nonphi_begin (void); + + // must label before id is valid + size_t id (void) const { return mid; } + + // dominance frontier + const df_set& df (void) const { return mdf; } + + df_iterator df_begin (void) const { return mdf.begin (); } + + df_iterator df_end (void) const { return mdf.end (); } + + // label with a RPO walk + void label (void) + { + size_t number = 0; + label (mvisit_count, number); + } + + void label (size_t avisit_count, size_t& number); + + // See for idom computation algorithm + // Cooper, Keith D.; Harvey, Timothy J; and Kennedy, Ken (2001). + // "A Simple, Fast Dominance Algorithm" + void compute_idom (jit_block& entry_block) + { + bool changed; + entry_block.idom = &entry_block; + do + changed = update_idom (mvisit_count); + while (changed); + } + + // compute dominance frontier + void compute_df (void) + { + compute_df (mvisit_count); + } + + void create_dom_tree (void) + { + create_dom_tree (mvisit_count); + } + + jit_block *dom_successor (size_t idx) const + { + return dom_succ[idx]; + } + + size_t dom_successor_count (void) const + { + return dom_succ.size (); + } + + // call pop_varaible on all instructions + void pop_all (void); + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; + + jit_block *maybe_split (jit_factory& factory, jit_block_list& blocks, + jit_block *asuccessor); + + jit_block *maybe_split (jit_factory& factory, jit_block_list& blocks, + jit_block& asuccessor) + { + return maybe_split (factory, blocks, &asuccessor); + } + + // print dominator infomration + std::ostream& print_dom (std::ostream& os) const; + + virtual std::ostream& short_print (std::ostream& os) const + { + os << mname; + if (mid != NO_ID) + os << mid; + else + os << "!"; + return os; + } + + llvm::BasicBlock *to_llvm (void) const; + + std::list<jit_block *>::iterator location (void) const + { return mlocation; } + + void stash_location (std::list<jit_block *>::iterator alocation) + { mlocation = alocation; } + + // used to prevent visiting the same node twice in the graph + size_t visit_count (void) const { return mvisit_count; } + + // check if this node has been visited yet at the given visit count. + // If we have not been visited yet, mark us as visited. + bool visited (size_t avisit_count) + { + if (mvisit_count <= avisit_count) + { + mvisit_count = avisit_count + 1; + return false; + } + + return true; + } + + jit_instruction *front (void) { return instructions.front (); } + + jit_instruction *back (void) { return instructions.back (); } + + JIT_VALUE_ACCEPT; +private: + void internal_append (jit_instruction *instr); + + void compute_df (size_t avisit_count); + + bool update_idom (size_t avisit_count); + + void create_dom_tree (size_t avisit_count); + + static jit_block *idom_intersect (jit_block *i, jit_block *j); + + size_t mvisit_count; + size_t mid; + jit_block *idom; + df_set mdf; + std::vector<jit_block *> dom_succ; + std::string mname; + instruction_list instructions; + bool malive; + std::list<jit_block *>::iterator mlocation; +}; + +// keeps track of phi functions that use a block on incomming edges +class +jit_phi_incomming : public jit_internal_node<jit_block, jit_phi_incomming> +{ +public: + jit_phi_incomming (void) : muser (0) { } + + jit_phi_incomming (jit_phi *auser) : muser (auser) { } + + jit_phi_incomming (const jit_phi_incomming& use) + { + *this = use; + } + + jit_phi_incomming& operator= (const jit_phi_incomming& use) + { + stash_value (use.value ()); + muser = use.muser; + return *this; + } + + jit_phi *user (void) const { return muser; } + + jit_block *user_parent (void) const; +private: + jit_phi *muser; +}; + +// A non-ssa variable +class +jit_variable : public jit_value +{ +public: + jit_variable (const std::string& aname) : mname (aname), mlast_use (0) { } + + const std::string &name (void) const { return mname; } + + // manipulate the value_stack, for use during SSA construction. The top of + // the value stack represents the current value for this variable + bool has_top (void) const + { + return ! value_stack.empty (); + } + + jit_value *top (void) const + { + return value_stack.top (); + } + + void push (jit_instruction *v) + { + value_stack.push (v); + mlast_use = v; + } + + void pop (void) + { + value_stack.pop (); + } + + jit_instruction *last_use (void) const + { + return mlast_use; + } + + void stash_last_use (jit_instruction *instr) + { + mlast_use = instr; + } + + // blocks in which we are used + void use_blocks (jit_block::df_set& result) + { + jit_use *use = first_use (); + while (use) + { + result.insert (use->user_parent ()); + use = use->next (); + } + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + return print_indent (os, indent) << mname; + } + + JIT_VALUE_ACCEPT; +private: + std::string mname; + std::stack<jit_value *> value_stack; + jit_instruction *mlast_use; +}; + +class +jit_assign_base : public jit_instruction +{ +public: + jit_assign_base (jit_variable *adest) : jit_instruction (), mdest (adest) { } + + jit_assign_base (jit_variable *adest, size_t npred) : jit_instruction (npred), + mdest (adest) { } + + jit_assign_base (jit_variable *adest, jit_value *arg0, jit_value *arg1) + : jit_instruction (arg0, arg1), mdest (adest) { } + + jit_variable *dest (void) const { return mdest; } + + virtual void push_variable (void) + { + mdest->push (this); + } + + virtual void pop_variable (void) + { + mdest->pop (); + } + + virtual std::ostream& short_print (std::ostream& os) const + { + if (type ()) + jit_print (os, type ()) << ": "; + + dest ()->short_print (os); + return os << "#" << id (); + } +private: + jit_variable *mdest; +}; + +class +jit_assign : public jit_assign_base +{ +public: + jit_assign (jit_variable *adest, jit_value *asrc) + : jit_assign_base (adest, adest, asrc), martificial (false) { } + + jit_value *overwrite (void) const + { + return argument (0); + } + + jit_value *src (void) const + { + return argument (1); + } + + // variables don't get modified in an SSA, but COW requires we modify + // variables. An artificial assign is for when a variable gets modified. We + // need an assign in the SSA, but the reference counts shouldn't be updated. + bool artificial (void) const { return martificial; } + + void mark_artificial (void) { martificial = true; } + + virtual bool infer (void) + { + jit_type *stype = src ()->type (); + if (stype != type()) + { + stash_type (stype); + return true; + } + + return false; + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent) << *this << " = " << *src (); + + if (artificial ()) + os << " [artificial]"; + + return os; + } + + JIT_VALUE_ACCEPT; +private: + bool martificial; +}; + +class +jit_phi : public jit_assign_base +{ +public: + jit_phi (jit_variable *adest, size_t npred) + : jit_assign_base (adest, npred) + { + mincomming.reserve (npred); + } + + // removes arguments form dead incomming jumps + bool prune (void); + + void add_incomming (jit_block *from, jit_value *value) + { + push_argument (value); + mincomming.push_back (jit_phi_incomming (this)); + mincomming[mincomming.size () - 1].stash_value (from); + } + + jit_block *incomming (size_t i) const + { + return mincomming[i].value (); + } + + llvm::BasicBlock *incomming_llvm (size_t i) const + { + return incomming (i)->to_llvm (); + } + + virtual void construct_ssa (void) { } + + virtual bool infer (void); + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + std::stringstream ss; + print_indent (ss, indent); + short_print (ss) << " phi "; + std::string ss_str = ss.str (); + std::string indent_str (ss_str.size (), ' '); + os << ss_str; + + for (size_t i = 0; i < argument_count (); ++i) + { + if (i > 0) + os << indent_str; + os << "| "; + + os << *incomming (i) << " -> "; + os << *argument (i); + + if (i + 1 < argument_count ()) + os << std::endl; + } + + return os; + } + + llvm::PHINode *to_llvm (void) const; + + JIT_VALUE_ACCEPT; +private: + std::vector<jit_phi_incomming> mincomming; +}; + +class +jit_terminator : public jit_instruction +{ +public: + +#define JIT_TERMINATOR_CONST(N) \ + jit_terminator (size_t asuccessor_count, \ + OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ + : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), \ + malive (asuccessor_count, false) { } + + JIT_TERMINATOR_CONST (1) + JIT_TERMINATOR_CONST (2) + JIT_TERMINATOR_CONST (3) + +#undef JIT_TERMINATOR_CONST + + jit_block *successor (size_t idx = 0) const + { + return static_cast<jit_block *> (argument (idx)); + } + + llvm::BasicBlock *successor_llvm (size_t idx = 0) const + { + return successor (idx)->to_llvm (); + } + + size_t successor_index (const jit_block *asuccessor) const; + + std::ostream& print_successor (std::ostream& os, size_t idx = 0) const + { + if (alive (idx)) + os << "[live] "; + else + os << "[dead] "; + + return successor (idx)->short_print (os); + } + + // Check if the jump to successor is live + bool alive (const jit_block *asuccessor) const + { + return alive (successor_index (asuccessor)); + } + + bool alive (size_t idx) const { return malive[idx]; } + + bool alive (int idx) const { return malive[idx]; } + + size_t successor_count (void) const { return malive.size (); } + + virtual bool infer (void); + + llvm::TerminatorInst *to_llvm (void) const; +protected: + virtual bool check_alive (size_t) const { return true; } +private: + std::vector<bool> malive; +}; + +class +jit_branch : public jit_terminator +{ +public: + jit_branch (jit_block *succ) : jit_terminator (1, succ) { } + + virtual size_t successor_count (void) const { return 1; } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent) << "branch: "; + return print_successor (os); + } + + JIT_VALUE_ACCEPT; +}; + +class +jit_cond_branch : public jit_terminator +{ +public: + jit_cond_branch (jit_value *c, jit_block *ctrue, jit_block *cfalse) + : jit_terminator (2, ctrue, cfalse, c) { } + + jit_value *cond (void) const { return argument (2); } + + std::ostream& print_cond (std::ostream& os) const + { + return cond ()->short_print (os); + } + + llvm::Value *cond_llvm (void) const + { + return cond ()->to_llvm (); + } + + virtual size_t successor_count (void) const { return 2; } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent) << "cond_branch: "; + print_cond (os) << ", "; + print_successor (os, 0) << ", "; + return print_successor (os, 1); + } + + JIT_VALUE_ACCEPT; +}; + +class +jit_call : public jit_instruction +{ +public: + jit_call (const jit_operation& (*aoperation) (void)) + : moperation (aoperation ()) + { + const jit_function& ol = overload (); + if (ol.valid ()) + stash_type (ol.result ()); + } + + jit_call (const jit_operation& aoperation) : moperation (aoperation) + { + const jit_function& ol = overload (); + if (ol.valid ()) + stash_type (ol.result ()); + } + +#define JIT_CALL_CONST(N) \ + jit_call (const jit_operation& aoperation, \ + OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ + : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), moperation (aoperation) { } \ + \ + jit_call (const jit_operation& (*aoperation) (void), \ + OCT_MAKE_DECL_LIST (jit_value *, arg, N)) \ + : jit_instruction (OCT_MAKE_ARG_LIST (arg, N)), moperation (aoperation ()) \ + { } + + JIT_CALL_CONST (1) + JIT_CALL_CONST (2) + JIT_CALL_CONST (3) + JIT_CALL_CONST (4) + +#undef JIT_CALL_CONST + + jit_call (const jit_operation& aoperation, + const std::vector<jit_value *>& args) + : jit_instruction (args), moperation (aoperation) + { } + + const jit_operation& operation (void) const { return moperation; } + + bool can_error (void) const + { + return overload ().can_error (); + } + + const jit_function& overload (void) const + { + return moperation.overload (argument_types ()); + } + + virtual bool needs_release (void) const; + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent); + + if (use_count ()) + short_print (os) << " = "; + os << "call " << moperation.name () << " ("; + + for (size_t i = 0; i < argument_count (); ++i) + { + print_argument (os, i); + if (i + 1 < argument_count ()) + os << ", "; + } + return os << ")"; + } + + virtual bool infer (void); + + JIT_VALUE_ACCEPT; +private: + const jit_operation& moperation; +}; + +// FIXME: This is just ugly... +// checks error_state, if error_state is false then goto the normal branch, +// otherwise goto the error branch +class +jit_error_check : public jit_terminator +{ +public: + // Which variable is the error check for? + enum variable + { + var_error_state, + var_interrupt + }; + + static std::string variable_to_string (variable v); + + jit_error_check (variable var, jit_call *acheck_for, jit_block *normal, + jit_block *error) + : jit_terminator (2, error, normal, acheck_for), mvariable (var) { } + + jit_error_check (variable var, jit_block *normal, jit_block *error) + : jit_terminator (2, error, normal), mvariable (var) { } + + variable check_variable (void) const { return mvariable; } + + bool has_check_for (void) const + { + return argument_count () == 3; + } + + jit_call *check_for (void) const + { + assert (has_check_for ()); + return static_cast<jit_call *> (argument (2)); + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; + + JIT_VALUE_ACCEPT; +protected: + virtual bool check_alive (size_t idx) const + { + if (! has_check_for ()) + return true; + return idx == 1 ? true : check_for ()->can_error (); + } +private: + variable mvariable; +}; + +// for now only handles the 1D case +class +jit_magic_end : public jit_instruction +{ +public: + class + context + { + public: + context (void) : value (0), index (0), count (0) + { } + + context (jit_factory& factory, jit_value *avalue, size_t aindex, + size_t acount); + + jit_value *value; + jit_const_index *index; + jit_const_index *count; + }; + + jit_magic_end (const std::vector<context>& full_context); + + virtual bool infer (void); + + const jit_function& overload () const; + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const; + + context resolve_context (void) const; + + virtual std::ostream& short_print (std::ostream& os) const + { + return os << "magic_end" << "#" << id (); + } + + JIT_VALUE_ACCEPT; +private: + std::vector<context> contexts; +}; + +class +jit_extract_argument : public jit_assign_base +{ +public: + jit_extract_argument (jit_type *atype, jit_variable *adest) + : jit_assign_base (adest) + { + stash_type (atype); + } + + const std::string& name (void) const + { + return dest ()->name (); + } + + const jit_function& overload (void) const + { + return jit_typeinfo::cast (type (), jit_typeinfo::get_any ()); + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent); + + return short_print (os) << " = extract " << name (); + } + + JIT_VALUE_ACCEPT; +}; + +class +jit_store_argument : public jit_instruction +{ +public: + jit_store_argument (jit_variable *var) + : jit_instruction (var), dest (var) + { } + + const std::string& name (void) const + { + return dest->name (); + } + + const jit_function& overload (void) const + { + return jit_typeinfo::cast (jit_typeinfo::get_any (), result_type ()); + } + + jit_value *result (void) const + { + return argument (0); + } + + jit_type *result_type (void) const + { + return result ()->type (); + } + + llvm::Value *result_llvm (void) const + { + return result ()->to_llvm (); + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + jit_value *res = result (); + print_indent (os, indent) << "store "; + dest->short_print (os); + + if (! isa<jit_variable> (res)) + { + os << " = "; + res->short_print (os); + } + + return os; + } + + JIT_VALUE_ACCEPT; +private: + jit_variable *dest; +}; + +class +jit_return : public jit_instruction +{ +public: + jit_return (void) { } + + jit_return (jit_value *retval) : jit_instruction (retval) { } + + jit_value *result (void) const + { + return argument_count () ? argument (0) : 0; + } + + jit_type *result_type (void) const + { + jit_value *res = result (); + return res ? res->type () : 0; + } + + virtual std::ostream& print (std::ostream& os, size_t indent = 0) const + { + print_indent (os, indent) << "return"; + + if (result ()) + os << " " << *result (); + + return os; + } + + JIT_VALUE_ACCEPT; +}; + +class +jit_ir_walker +{ +public: + virtual ~jit_ir_walker () { } + +#define JIT_METH(clname) \ + virtual void visit (jit_ ## clname&) = 0; + + JIT_VISIT_IR_CLASSES; + +#undef JIT_METH +}; + +template <typename T, jit_type *(*EXTRACT_T)(void), typename PASS_T, bool QUOTE> +void +jit_const<T, EXTRACT_T, PASS_T, QUOTE>::accept (jit_ir_walker& walker) +{ + walker.visit (*this); +} + +#undef JIT_VALUE_ACCEPT + +#endif +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-typeinfo.cc Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,2243 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +// defines required by llvm +#define __STDC_LIMIT_MACROS +#define __STDC_CONSTANT_MACROS + +#if defined (HAVE_CONFIG_H) +# include "config.h" +#endif + +#if defined (HAVE_LLVM) + +#include "jit-typeinfo.h" + +#if defined (HAVE_LLVM_IR_VERIFIER_H) +# include <llvm/IR/Verifier.h> +#else +# include <llvm/Analysis/Verifier.h> +#endif + +#include <llvm/ExecutionEngine/ExecutionEngine.h> + +#if defined (HAVE_LLVM_IR_FUNCTION_H) +# include <llvm/IR/GlobalVariable.h> +# include <llvm/IR/LLVMContext.h> +# include <llvm/IR/Function.h> +# include <llvm/IR/Instructions.h> +# include <llvm/IR/Intrinsics.h> +#else +# include <llvm/GlobalVariable.h> +# include <llvm/LLVMContext.h> +# include <llvm/Function.h> +# include <llvm/Instructions.h> +# include <llvm/Intrinsics.h> +#endif + +#if defined (HAVE_LLVM_SUPPORT_IRBUILDER_H) +# include <llvm/Support/IRBuilder.h> +# elif defined(HAVE_LLVM_IR_IRBUILDER_H) +# include <llvm/IR/IRBuilder.h> +#else +# include <llvm/IRBuilder.h> +#endif + +#include <llvm/Support/raw_os_ostream.h> + +#include "jit-ir.h" +#include "ov.h" +#include "ov-builtin.h" +#include "ov-complex.h" +#include "ov-scalar.h" +#include "pager.h" + +static llvm::LLVMContext& context = llvm::getGlobalContext (); + +jit_typeinfo *jit_typeinfo::instance = 0; + +std::ostream& jit_print (std::ostream& os, jit_type *atype) +{ + if (! atype) + return os << "null"; + return os << atype->name (); +} + +// function that jit code calls +extern "C" void +octave_jit_print_any (const char *name, octave_base_value *obv) +{ + obv->print_with_name (octave_stdout, name, true); +} + +extern "C" void +octave_jit_print_scalar (const char *name, double value) +{ + // FIXME: We should avoid allocating a new octave_scalar each time + octave_value ov (value); + ov.print_with_name (octave_stdout, name); +} + +extern "C" octave_base_value* +octave_jit_binary_any_any (octave_value::binary_op op, octave_base_value *lhs, + octave_base_value *rhs) +{ + octave_value olhs (lhs, true); + octave_value orhs (rhs, true); + octave_value result = do_binary_op (op, olhs, orhs); + octave_base_value *rep = result.internal_rep (); + rep->grab (); + return rep; +} + +extern "C" octave_idx_type +octave_jit_compute_nelem (double base, double limit, double inc) +{ + Range rng = Range (base, limit, inc); + return rng.numel (); +} + +extern "C" void +octave_jit_release_any (octave_base_value *obv) +{ + obv->release (); +} + +extern "C" void +octave_jit_release_matrix (jit_matrix *m) +{ + delete m->array; +} + +extern "C" octave_base_value * +octave_jit_grab_any (octave_base_value *obv) +{ + obv->grab (); + return obv; +} + +extern "C" jit_matrix +octave_jit_grab_matrix (jit_matrix *m) +{ + return *m->array; +} + +extern "C" octave_base_value * +octave_jit_cast_any_matrix (jit_matrix *m) +{ + octave_value ret (*m->array); + octave_base_value *rep = ret.internal_rep (); + rep->grab (); + delete m->array; + + return rep; +} + +extern "C" jit_matrix +octave_jit_cast_matrix_any (octave_base_value *obv) +{ + NDArray m = obv->array_value (); + obv->release (); + return m; +} + +extern "C" octave_base_value * +octave_jit_cast_any_range (jit_range *rng) +{ + Range temp (*rng); + octave_value ret (temp); + octave_base_value *rep = ret.internal_rep (); + rep->grab (); + + return rep; +} +extern "C" jit_range +octave_jit_cast_range_any (octave_base_value *obv) +{ + + jit_range r (obv->range_value ()); + obv->release (); + return r; +} + +extern "C" double +octave_jit_cast_scalar_any (octave_base_value *obv) +{ + double ret = obv->double_value (); + obv->release (); + return ret; +} + +extern "C" octave_base_value * +octave_jit_cast_any_scalar (double value) +{ + return new octave_scalar (value); +} + +extern "C" Complex +octave_jit_cast_complex_any (octave_base_value *obv) +{ + Complex ret = obv->complex_value (); + obv->release (); + return ret; +} + +extern "C" octave_base_value * +octave_jit_cast_any_complex (Complex c) +{ + if (c.imag () == 0) + return new octave_scalar (c.real ()); + else + return new octave_complex (c); +} + +extern "C" void +octave_jit_octave::err_nan_to_logical_conversion (void) +{ + octave::err_nan_to_logical_conversion (); +} + +extern "C" void +octave_jit_ginvalid_index (void) +{ + // FIXME: 0-argument form of octave::err_invalid_index removed in cset dd6345fd8a97 + // Report -1 as the bad index for all occurrences. + octave::err_invalid_index (static_cast<octave_idx_type> (-1)); +} + +extern "C" void +octave_jit_gindex_range (int nd, int dim, octave_idx_type iext, + octave_idx_type ext) +{ + octave::err_index_out_of_range (nd, dim, iext, ext); +} + +extern "C" jit_matrix +octave_jit_paren_subsasgn_impl (jit_matrix *mat, octave_idx_type index, + double value) +{ + NDArray *array = mat->array; + if (array->numel () < index) + array->resize1 (index); + + double *data = array->fortran_vec (); + data[index - 1] = value; + + mat->update (); + return *mat; +} + +static void +make_indices (double *indices, octave_idx_type idx_count, + Array<idx_vector>& result) +{ + result.resize (dim_vector (1, idx_count)); + for (octave_idx_type i = 0; i < idx_count; ++i) + result(i) = idx_vector (indices[i]); +} + +extern "C" double +octave_jit_paren_scalar (jit_matrix *mat, double *indicies, + octave_idx_type idx_count) +{ + // FIXME: Replace this with a more optimal version + Array<idx_vector> idx; + make_indices (indicies, idx_count, idx); + + Array<double> ret = mat->array->index (idx); + + return ret.xelem (0); +} + +extern "C" jit_matrix +octave_jit_paren_scalar_subsasgn (jit_matrix *mat, double *indices, + octave_idx_type idx_count, double value) +{ + // FIXME: Replace this with a more optimal version + jit_matrix ret; + + Array<idx_vector> idx; + make_indices (indices, idx_count, idx); + + Matrix temp (1, 1); + temp.xelem(0) = value; + mat->array->assign (idx, temp); + ret.update (mat->array); + + return ret; +} + +extern "C" jit_matrix +octave_jit_paren_subsasgn_matrix_range (jit_matrix *mat, jit_range *index, + double value) +{ + NDArray *array = mat->array; + bool done = false; + + // optimize for the simple case (no resizing and no errors) + if (*array->jit_ref_count () == 1 + && index->all_elements_are_ints ()) + { + // this code is similar to idx_vector::fill, but we avoid allocating an + // idx_vector and its associated rep + octave_idx_type start = static_cast<octave_idx_type> (index->base) - 1; + octave_idx_type step = static_cast<octave_idx_type> (index->inc); + octave_idx_type nelem = index->nelem; + octave_idx_type final = start + nelem * step; + if (step < 0) + { + step = -step; + std::swap (final, start); + } + + if (start >= 0 && final < mat->slice_len) + { + done = true; + + double *data = array->jit_slice_data (); + if (step == 1) + std::fill (data + start, data + start + nelem, value); + else + { + for (octave_idx_type i = start; i < final; i += step) + data[i] = value; + } + } + } + + if (! done) + { + idx_vector idx (*index); + NDArray avalue (dim_vector (1, 1)); + avalue.xelem (0) = value; + array->assign (idx, avalue); + } + + jit_matrix ret; + ret.update (array); + return ret; +} + +extern "C" double +octave_jit_end_matrix (jit_matrix *mat, octave_idx_type idx, + octave_idx_type count) +{ + octave_idx_type ndim = mat->dimensions[-1]; + if (ndim == count) + return mat->dimensions[idx]; + else if (ndim > count) + { + if (idx == count - 1) + { + double ret = mat->dimensions[idx]; + for (octave_idx_type i = idx + 1; i < ndim; ++i) + ret *= mat->dimensions[idx]; + return ret; + } + + return mat->dimensions[idx]; + } + else // ndim < count + return idx < ndim ? mat->dimensions[idx] : 1; +} + +extern "C" octave_base_value * +octave_jit_create_undef (void) +{ + octave_value undef; + octave_base_value *ret = undef.internal_rep (); + ret->grab (); + + return ret; +} + +extern "C" Complex +octave_jit_complex_mul (Complex lhs, Complex rhs) +{ + if (lhs.imag () == 0 && rhs.imag() == 0) + return Complex (lhs.real () * rhs.real (), 0); + + return lhs * rhs; +} + +extern "C" Complex +octave_jit_complex_div (Complex lhs, Complex rhs) +{ + // see src/OPERATORS/op-cs-cs.cc + if (rhs == 0.0) + warn_divide_by_zero (); + + return lhs / rhs; +} + +// FIXME: CP form src/xpow.cc +static inline int +xisint (double x) +{ + return (octave::math::x_nint (x) == x + && ((x >= 0 && x < std::numeric_limits<int>::max ()) + || (x <= 0 && x > std::numeric_limits<int>::min ()))); +} + +extern "C" Complex +octave_jit_pow_scalar_scalar (double lhs, double rhs) +{ + // FIXME: almost CP from src/xpow.cc + if (lhs < 0.0 && ! xisint (rhs)) + return std::pow (Complex (lhs), rhs); + return std::pow (lhs, rhs); +} + +extern "C" Complex +octave_jit_pow_complex_complex (Complex lhs, Complex rhs) +{ + if (lhs.imag () == 0 && rhs.imag () == 0) + return octave_jit_pow_scalar_scalar (lhs.real (), rhs.real ()); + return std::pow (lhs, rhs); +} + +extern "C" Complex +octave_jit_pow_complex_scalar (Complex lhs, double rhs) +{ + if (lhs.imag () == 0) + return octave_jit_pow_scalar_scalar (lhs.real (), rhs); + return std::pow (lhs, rhs); +} + +extern "C" Complex +octave_jit_pow_scalar_complex (double lhs, Complex rhs) +{ + if (rhs.imag () == 0) + return octave_jit_pow_scalar_scalar (lhs, rhs.real ()); + return std::pow (lhs, rhs); +} + +extern "C" void +octave_jit_print_matrix (jit_matrix *m) +{ + std::cout << *m << std::endl; +} + +OCTAVE_NORETURN static +void +err_bad_result (void) +{ + error ("incorrect type information given to the JIT compiler"); +} + +// FIXME: Add support for multiple outputs +extern "C" octave_base_value * +octave_jit_call (octave_builtin::fcn fn, size_t nargin, + octave_base_value **argin, jit_type *result_type) +{ + octave_value_list ovl (nargin); + for (size_t i = 0; i < nargin; ++i) + ovl.xelem (i) = octave_value (argin[i]); + + ovl = fn (ovl, 1); + + // FIXME: Check result_type somehow + if (result_type) + { + if (ovl.length () < 1) + err_bad_result (); + + octave_value result = ovl.xelem(0); + octave_base_value *ret = result.internal_rep (); + ret->grab (); + return ret; + } + + if (! (ovl.empty () + || (ovl.length () == 1 && ovl.xelem (0).is_undefined ()))) + err_bad_result (); + + return 0; +} + +// -------------------- jit_range -------------------- +bool +jit_range::all_elements_are_ints () const +{ + Range r (*this); + return r.all_elements_are_ints (); +} + +std::ostream& +operator<< (std::ostream& os, const jit_range& rng) +{ + return os << "Range[" << rng.base << ", " << rng.limit << ", " << rng.inc + << ", " << rng.nelem << "]"; +} + +// -------------------- jit_matrix -------------------- + +std::ostream& +operator<< (std::ostream& os, const jit_matrix& mat) +{ + return os << "Matrix[" << mat.ref_count << ", " << mat.slice_data << ", " + << mat.slice_len << ", " << mat.dimensions << ", " + << mat.array << "]"; +} + +// -------------------- jit_type -------------------- +jit_type::jit_type (const std::string& aname, jit_type *aparent, + llvm::Type *allvm_type, bool askip_paren, int aid) : + mname (aname), mparent (aparent), llvm_type (allvm_type), mid (aid), + mdepth (aparent ? aparent->mdepth + 1 : 0), mskip_paren (askip_paren) +{ + std::memset (msret, 0, sizeof (msret)); + std::memset (mpointer_arg, 0, sizeof (mpointer_arg)); + std::memset (mpack, 0, sizeof (mpack)); + std::memset (munpack, 0, sizeof (munpack)); + + for (size_t i = 0; i < jit_convention::length; ++i) + mpacked_type[i] = llvm_type; +} + +llvm::Type * +jit_type::to_llvm_arg (void) const +{ + return llvm_type ? llvm_type->getPointerTo () : 0; +} + +// -------------------- jit_function -------------------- +jit_function::jit_function () : module (0), llvm_function (0), mresult (0), + call_conv (jit_convention::length), + mcan_error (false) +{ } + +jit_function::jit_function (llvm::Module *amodule, + jit_convention::type acall_conv, + const llvm::Twine& aname, jit_type *aresult, + const std::vector<jit_type *>& aargs) + : module (amodule), mresult (aresult), args (aargs), call_conv (acall_conv), + mcan_error (false) +{ + llvm::SmallVector<llvm::Type *, 15> llvm_args; + + llvm::Type *rtype = llvm::Type::getVoidTy (context); + if (mresult) + { + rtype = mresult->packed_type (call_conv); + if (sret ()) + { + llvm_args.push_back (rtype->getPointerTo ()); + rtype = llvm::Type::getVoidTy (context); + } + } + + for (std::vector<jit_type *>::const_iterator iter = args.begin (); + iter != args.end (); ++iter) + { + jit_type *ty = *iter; + assert (ty); + llvm::Type *argty = ty->packed_type (call_conv); + if (ty->pointer_arg (call_conv)) + argty = argty->getPointerTo (); + + llvm_args.push_back (argty); + } + + // we mark all functinos as external linkage because this prevents llvm + // from getting rid of always inline functions + llvm::FunctionType *ft = llvm::FunctionType::get (rtype, llvm_args, false); + llvm_function = llvm::Function::Create (ft, llvm::Function::ExternalLinkage, + aname, module); + + if (sret ()) + { +#if defined (FUNCTION_ADDATTRIBUTE_ARG_IS_ATTRIBUTES) + llvm::AttrBuilder attr_builder; + attr_builder.addAttribute (llvm::Attributes::StructRet); + llvm::Attributes attrs = llvm::Attributes::get(context, attr_builder); + llvm_function->addAttribute (1, attrs); +#else + llvm_function->addAttribute (1, llvm::Attribute::StructRet); +#endif + } + + if (call_conv == jit_convention::internal) +#if defined (FUNCTION_ADDFNATTR_ARG_IS_ATTRIBUTES) + llvm_function->addFnAttr (llvm::Attributes::AlwaysInline); +#else + llvm_function->addFnAttr (llvm::Attribute::AlwaysInline); +#endif +} + +jit_function::jit_function (const jit_function& fn, jit_type *aresult, + const std::vector<jit_type *>& aargs) + : module (fn.module), llvm_function (fn.llvm_function), mresult (aresult), + args (aargs), call_conv (fn.call_conv), mcan_error (fn.mcan_error) +{ } + +jit_function::jit_function (const jit_function& fn) + : module (fn.module), llvm_function (fn.llvm_function), mresult (fn.mresult), + args (fn.args), call_conv (fn.call_conv), mcan_error (fn.mcan_error) +{ } + +void +jit_function::erase (void) +{ + if (! llvm_function) + return; + + llvm_function->eraseFromParent (); + llvm_function = 0; +} + +std::string +jit_function::name (void) const +{ + return llvm_function->getName (); +} + +llvm::BasicBlock * +jit_function::new_block (const std::string& aname, + llvm::BasicBlock *insert_before) +{ + return llvm::BasicBlock::Create (context, aname, llvm_function, + insert_before); +} + +llvm::Value * +jit_function::call (llvm::IRBuilderD& builder, + const std::vector<jit_value *>& in_args) const +{ + if (! valid ()) + throw jit_fail_exception ("Call not implemented"); + + assert (in_args.size () == args.size ()); + std::vector<llvm::Value *> llvm_args (args.size ()); + for (size_t i = 0; i < in_args.size (); ++i) + llvm_args[i] = in_args[i]->to_llvm (); + + return call (builder, llvm_args); +} + +llvm::Value * +jit_function::call (llvm::IRBuilderD& builder, + const std::vector<llvm::Value *>& in_args) const +{ + if (! valid ()) + throw jit_fail_exception ("Call not implemented"); + + assert (in_args.size () == args.size ()); + llvm::SmallVector<llvm::Value *, 10> llvm_args; + llvm_args.reserve (in_args.size () + sret ()); + + llvm::BasicBlock *insert_block = builder.GetInsertBlock (); + llvm::Function *parent = insert_block->getParent (); + assert (parent); + + // we insert allocas inside the prelude block to prevent stack overflows + llvm::BasicBlock& prelude = parent->getEntryBlock (); + llvm::IRBuilder<> pre_builder (&prelude, prelude.begin ()); + + llvm::AllocaInst *sret_mem = 0; + if (sret ()) + { + sret_mem = pre_builder.CreateAlloca (mresult->packed_type (call_conv)); + llvm_args.push_back (sret_mem); + } + + for (size_t i = 0; i < in_args.size (); ++i) + { + llvm::Value *arg = in_args[i]; + jit_type::convert_fn convert = args[i]->pack (call_conv); + if (convert) + arg = convert (builder, arg); + + if (args[i]->pointer_arg (call_conv)) + { + llvm::Type *ty = args[i]->packed_type (call_conv); + llvm::Value *alloca = pre_builder.CreateAlloca (ty); + builder.CreateStore (arg, alloca); + arg = alloca; + } + + llvm_args.push_back (arg); + } + + llvm::CallInst *callinst = builder.CreateCall (llvm_function, llvm_args); + llvm::Value *ret = callinst; + + if (sret ()) + { +#if defined (CALLINST_ADDATTRIBUTE_ARG_IS_ATTRIBUTES) + llvm::AttrBuilder attr_builder; + attr_builder.addAttribute(llvm::Attributes::StructRet); + llvm::Attributes attrs = llvm::Attributes::get(context, attr_builder); + callinst->addAttribute (1, attrs); +#else + callinst->addAttribute (1, llvm::Attribute::StructRet); +#endif + ret = builder.CreateLoad (sret_mem); + } + + if (mresult) + { + jit_type::convert_fn unpack = mresult->unpack (call_conv); + if (unpack) + ret = unpack (builder, ret); + } + + return ret; +} + +llvm::Value * +jit_function::argument (llvm::IRBuilderD& builder, size_t idx) const +{ + assert (idx < args.size ()); + + // FIXME: We should be treating arguments like a list, not a vector. + // Shouldn't matter much for now, as the number of arguments shouldn't + // be much bigger than 4 + llvm::Function::arg_iterator iter = llvm_function->arg_begin (); + if (sret ()) + ++iter; + + for (size_t i = 0; i < idx; ++i, ++iter); + + if (args[idx]->pointer_arg (call_conv)) + return builder.CreateLoad (iter); + + return iter; +} + +void +jit_function::do_return (llvm::IRBuilderD& builder, llvm::Value *rval, + bool verify) +{ + assert (! rval == ! mresult); + + if (rval) + { + jit_type::convert_fn convert = mresult->pack (call_conv); + if (convert) + rval = convert (builder, rval); + + if (sret ()) + { + builder.CreateStore (rval, llvm_function->arg_begin ()); + builder.CreateRetVoid (); + } + else + builder.CreateRet (rval); + } + else + builder.CreateRetVoid (); + + if (verify) + llvm::verifyFunction (*llvm_function); +} + +void +jit_function::do_add_mapping (llvm::ExecutionEngine *engine, void *fn) +{ + assert (valid ()); + engine->addGlobalMapping (llvm_function, fn); +} + +std::ostream& +operator<< (std::ostream& os, const jit_function& fn) +{ + llvm::Function *lfn = fn.to_llvm (); + os << "jit_function: cc=" << fn.call_conv; + llvm::raw_os_ostream llvm_out (os); + lfn->print (llvm_out); + llvm_out.flush (); + return os; +} + +// -------------------- jit_operation -------------------- +jit_operation::~jit_operation (void) +{ + for (generated_map::iterator iter = generated.begin (); + iter != generated.end (); ++iter) + { + delete iter->first; + delete iter->second; + } +} + +void +jit_operation::add_overload (const jit_function& func, + const std::vector<jit_type*>& args) +{ + if (args.size () >= overloads.size ()) + overloads.resize (args.size () + 1); + + Array<jit_function>& over = overloads[args.size ()]; + dim_vector dv (over.dims ()); + Array<octave_idx_type> idx = to_idx (args); + bool must_resize = false; + + if (dv.length () != idx.numel ()) + { + dv.resize (idx.numel ()); + must_resize = true; + } + + for (octave_idx_type i = 0; i < dv.length (); ++i) + if (dv(i) <= idx(i)) + { + must_resize = true; + dv(i) = idx(i) + 1; + } + + if (must_resize) + over.resize (dv); + + over(idx) = func; +} + +const jit_function& +jit_operation::overload (const std::vector<jit_type*>& types) const +{ + static jit_function null_overload; + for (size_t i = 0; i < types.size (); ++i) + if (! types[i]) + return null_overload; + + if (types.size () >= overloads.size ()) + return do_generate (types); + + const Array<jit_function>& over = overloads[types.size ()]; + dim_vector dv (over.dims ()); + Array<octave_idx_type> idx = to_idx (types); + for (octave_idx_type i = 0; i < dv.length (); ++i) + if (idx(i) >= dv(i)) + return do_generate (types); + + const jit_function& ret = over(idx); + if (! ret.valid ()) + return do_generate (types); + + return ret; +} + +Array<octave_idx_type> +jit_operation::to_idx (const std::vector<jit_type*>& types) const +{ + octave_idx_type numel = types.size (); + numel = std::max (numel, static_cast<octave_idx_type>(2)); + + Array<octave_idx_type> idx (dim_vector (1, numel)); + for (octave_idx_type i = 0; i < static_cast<octave_idx_type> (types.size ()); + ++i) + idx(i) = types[i]->type_id (); + + if (types.size () == 0) + idx(0) = idx(1) = 0; + if (types.size () == 1) + { + idx(1) = idx(0); + idx(0) = 0; + } + + return idx; +} + +const jit_function& +jit_operation::do_generate (const signature_vec& types) const +{ + static jit_function null_overload; + generated_map::const_iterator find = generated.find (&types); + if (find != generated.end ()) + { + if (find->second) + return *find->second; + else + return null_overload; + } + + jit_function *ret = generate (types); + generated[new signature_vec (types)] = ret; + return ret ? *ret : null_overload; +} + +jit_function * +jit_operation::generate (const signature_vec&) const +{ + return 0; +} + +bool +jit_operation::signature_cmp::operator () (const signature_vec *lhs, + const signature_vec *rhs) const +{ + const signature_vec& l = *lhs; + const signature_vec& r = *rhs; + + if (l.size () < r.size ()) + return true; + else if (l.size () > r.size ()) + return false; + + for (size_t i = 0; i < l.size (); ++i) + { + if (l[i]->type_id () < r[i]->type_id ()) + return true; + else if (l[i]->type_id () > r[i]->type_id ()) + return false; + } + + return false; +} + +// -------------------- jit_index_operation -------------------- +jit_function * +jit_index_operation::generate (const signature_vec& types) const +{ + if (types.size () > 2 && types[0] == jit_typeinfo::get_matrix ()) + { + // indexing a matrix with scalars + jit_type *scalar = jit_typeinfo::get_scalar (); + for (size_t i = 1; i < types.size (); ++i) + if (types[i] != scalar) + return 0; + + return generate_matrix (types); + } + + return 0; +} + +llvm::Value * +jit_index_operation::create_arg_array (llvm::IRBuilderD& builder, + const jit_function &fn, size_t start_idx, + size_t end_idx) const +{ + size_t n = end_idx - start_idx; + llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); + llvm::ArrayType *array_t = llvm::ArrayType::get (scalar_t, n); + llvm::Value *array = llvm::UndefValue::get (array_t); + for (size_t i = start_idx; i < end_idx; ++i) + { + llvm::Value *idx = fn.argument (builder, i); + array = builder.CreateInsertValue (array, idx, i - start_idx); + } + + llvm::Value *array_mem = builder.CreateAlloca (array_t); + builder.CreateStore (array, array_mem); + return builder.CreateBitCast (array_mem, scalar_t->getPointerTo ()); +} + +// -------------------- jit_paren_subsref -------------------- +jit_function * +jit_paren_subsref::generate_matrix (const signature_vec& types) const +{ + std::stringstream ss; + ss << "jit_paren_subsref_matrix_scalar" << (types.size () - 1); + + jit_type *scalar = jit_typeinfo::get_scalar (); + jit_function *fn = new jit_function (module, jit_convention::internal, + ss.str (), scalar, types); + fn->mark_can_error (); + llvm::BasicBlock *body = fn->new_block (); + llvm::IRBuilder<> builder (body); + + llvm::Value *array = create_arg_array (builder, *fn, 1, types.size ()); + jit_type *index = jit_typeinfo::get_index (); + llvm::Value *nelem = llvm::ConstantInt::get (index->to_llvm (), + types.size () - 1); + llvm::Value *mat = fn->argument (builder, 0); + llvm::Value *ret = paren_scalar.call (builder, mat, array, nelem); + fn->do_return (builder, ret); + return fn; +} + +void +jit_paren_subsref::do_initialize (void) +{ + std::vector<jit_type *> types (3); + types[0] = jit_typeinfo::get_matrix (); + types[1] = jit_typeinfo::get_scalar_ptr (); + types[2] = jit_typeinfo::get_index (); + + jit_type *scalar = jit_typeinfo::get_scalar (); + paren_scalar = jit_function (module, jit_convention::external, + "octave_jit_paren_scalar", scalar, types); + paren_scalar.add_mapping (engine, &octave_jit_paren_scalar); + paren_scalar.mark_can_error (); +} + +// -------------------- jit_paren_subsasgn -------------------- +jit_function * +jit_paren_subsasgn::generate_matrix (const signature_vec& types) const +{ + std::stringstream ss; + ss << "jit_paren_subsasgn_matrix_scalar" << (types.size () - 2); + + jit_type *matrix = jit_typeinfo::get_matrix (); + jit_function *fn = new jit_function (module, jit_convention::internal, + ss.str (), matrix, types); + fn->mark_can_error (); + llvm::BasicBlock *body = fn->new_block (); + llvm::IRBuilder<> builder (body); + + llvm::Value *array = create_arg_array (builder, *fn, 1, types.size () - 1); + jit_type *index = jit_typeinfo::get_index (); + llvm::Value *nelem = llvm::ConstantInt::get (index->to_llvm (), + types.size () - 2); + + llvm::Value *mat = fn->argument (builder, 0); + llvm::Value *value = fn->argument (builder, types.size () - 1); + llvm::Value *ret = paren_scalar.call (builder, mat, array, nelem, value); + fn->do_return (builder, ret); + return fn; +} + +void +jit_paren_subsasgn::do_initialize (void) +{ + if (paren_scalar.valid ()) + return; + + jit_type *matrix = jit_typeinfo::get_matrix (); + std::vector<jit_type *> types (4); + types[0] = matrix; + types[1] = jit_typeinfo::get_scalar_ptr (); + types[2] = jit_typeinfo::get_index (); + types[3] = jit_typeinfo::get_scalar (); + + paren_scalar = jit_function (module, jit_convention::external, + "octave_jit_paren_scalar", matrix, types); + paren_scalar.add_mapping (engine, &octave_jit_paren_scalar_subsasgn); + paren_scalar.mark_can_error (); +} + +// -------------------- jit_typeinfo -------------------- +void +jit_typeinfo::initialize (llvm::Module *m, llvm::ExecutionEngine *e) +{ + new jit_typeinfo (m, e); +} + +// wrap function names to simplify jit_typeinfo::create_external +#define JIT_FN(fn) engine, &fn, #fn + +jit_typeinfo::jit_typeinfo (llvm::Module *m, llvm::ExecutionEngine *e) + : module (m), engine (e), next_id (0), + builder (*new llvm::IRBuilderD (context)) +{ + instance = this; + + // FIXME: We should be registering types like in octave_value_typeinfo + llvm::Type *any_t = llvm::StructType::create (context, "octave_base_value"); + any_t = any_t->getPointerTo (); + + llvm::Type *scalar_t = llvm::Type::getDoubleTy (context); + llvm::Type *bool_t = llvm::Type::getInt1Ty (context); + llvm::Type *string_t = llvm::Type::getInt8Ty (context); + string_t = string_t->getPointerTo (); + llvm::Type *index_t = llvm::Type::getIntNTy (context, + sizeof(octave_idx_type) * 8); + + llvm::StructType *range_t = llvm::StructType::create (context, "range"); + std::vector<llvm::Type *> range_contents (4, scalar_t); + range_contents[3] = index_t; + range_t->setBody (range_contents); + + llvm::Type *refcount_t = llvm::Type::getIntNTy (context, sizeof(int) * 8); + + llvm::StructType *matrix_t = llvm::StructType::create (context, "matrix"); + llvm::Type *matrix_contents[5]; + matrix_contents[0] = refcount_t->getPointerTo (); + matrix_contents[1] = scalar_t->getPointerTo (); + matrix_contents[2] = index_t; + matrix_contents[3] = index_t->getPointerTo (); + matrix_contents[4] = string_t; + matrix_t->setBody (llvm::makeArrayRef (matrix_contents, 5)); + + llvm::Type *complex_t = llvm::ArrayType::get (scalar_t, 2); + + // complex_ret is what is passed to C functions in order to get calling + // convention right + llvm::Type *cmplx_inner_cont[] = {scalar_t, scalar_t}; + llvm::StructType *cmplx_inner = llvm::StructType::create (cmplx_inner_cont); + + complex_ret = llvm::StructType::create (context, "complex_ret"); + { + llvm::Type *contents[] = {cmplx_inner}; + complex_ret->setBody (contents); + } + + // create types + any = new_type ("any", 0, any_t); + matrix = new_type ("matrix", any, matrix_t); + complex = new_type ("complex", any, complex_t); + scalar = new_type ("scalar", complex, scalar_t); + scalar_ptr = new_type ("scalar_ptr", 0, scalar_t->getPointerTo ()); + any_ptr = new_type ("any_ptr", 0, any_t->getPointerTo ()); + range = new_type ("range", any, range_t); + string = new_type ("string", any, string_t); + boolean = new_type ("bool", any, bool_t); + index = new_type ("index", any, index_t); + + create_int (8); + create_int (16); + create_int (32); + create_int (64); + + casts.resize (next_id + 1); + identities.resize (next_id + 1); + + // specify calling conventions + // FIXME: We should detect architecture and do something sane based on that + // here we assume x86 or x86_64 + matrix->mark_sret (jit_convention::external); + matrix->mark_pointer_arg (jit_convention::external); + + range->mark_sret (jit_convention::external); + range->mark_pointer_arg (jit_convention::external); + + complex->set_pack (jit_convention::external, &jit_typeinfo::pack_complex); + complex->set_unpack (jit_convention::external, &jit_typeinfo::unpack_complex); + complex->set_packed_type (jit_convention::external, complex_ret); + + if (sizeof (void *) == 4) + complex->mark_sret (jit_convention::external); + + paren_subsref_fn.initialize (module, engine); + paren_subsasgn_fn.initialize (module, engine); + + // bind global variables + lerror_state = new llvm::GlobalVariable (*module, bool_t, false, + llvm::GlobalValue::ExternalLinkage, + 0, "error_state"); + engine->addGlobalMapping (lerror_state, + reinterpret_cast<void *> (&error_state)); + + // sig_atomic_type is going to be some sort of integer + sig_atomic_type = llvm::Type::getIntNTy (context, sizeof(sig_atomic_t) * 8); + loctave_interrupt_state + = new llvm::GlobalVariable (*module, sig_atomic_type, false, + llvm::GlobalValue::ExternalLinkage, 0, + "octave_interrupt_state"); + engine->addGlobalMapping (loctave_interrupt_state, + reinterpret_cast<void *> (&octave_interrupt_state)); + + // generic call function + { + jit_type *int_t = intN (sizeof (octave_builtin::fcn) * 8); + any_call = create_external (JIT_FN (octave_jit_call), any, int_t, int_t, + any_ptr, int_t); + } + + // any with anything is an any op + jit_function fn; + jit_type *binary_op_type = intN (sizeof (octave_value::binary_op) * 8); + llvm::Type *llvm_bo_type = binary_op_type->to_llvm (); + jit_function any_binary = create_external (JIT_FN (octave_jit_binary_any_any), + any, binary_op_type, any, any); + any_binary.mark_can_error (); + binary_ops.resize (octave_value::num_binary_ops); + for (size_t i = 0; i < octave_value::num_binary_ops; ++i) + { + octave_value::binary_op op = static_cast<octave_value::binary_op> (i); + std::string op_name = octave_value::binary_op_as_string (op); + binary_ops[i].stash_name ("binary" + op_name); + } + + unary_ops.resize (octave_value::num_unary_ops); + for (size_t i = 0; i < octave_value::num_unary_ops; ++i) + { + octave_value::unary_op op = static_cast<octave_value::unary_op> (i); + std::string op_name = octave_value::unary_op_as_string (op); + unary_ops[i].stash_name ("unary" + op_name); + } + + for (int op = 0; op < octave_value::num_binary_ops; ++op) + { + llvm::Twine fn_name ("octave_jit_binary_any_any_"); + fn_name = fn_name + llvm::Twine (op); + + fn = create_internal (fn_name, any, any, any); + fn.mark_can_error (); + llvm::BasicBlock *block = fn.new_block (); + builder.SetInsertPoint (block); + llvm::APInt op_int(sizeof (octave_value::binary_op) * 8, op, + std::numeric_limits<octave_value::binary_op>::is_signed); + llvm::Value *op_as_llvm = llvm::ConstantInt::get (llvm_bo_type, op_int); + llvm::Value *ret = any_binary.call (builder, op_as_llvm, + fn.argument (builder, 0), + fn.argument (builder, 1)); + fn.do_return (builder, ret); + binary_ops[op].add_overload (fn); + } + + // grab matrix + fn = create_external (JIT_FN (octave_jit_grab_matrix), matrix, matrix); + grab_fn.add_overload (fn); + + grab_fn.add_overload (create_identity (scalar)); + grab_fn.add_overload (create_identity (scalar_ptr)); + grab_fn.add_overload (create_identity (any_ptr)); + grab_fn.add_overload (create_identity (boolean)); + grab_fn.add_overload (create_identity (complex)); + grab_fn.add_overload (create_identity (index)); + + // release any + fn = create_external (JIT_FN (octave_jit_release_any), 0, any); + release_fn.add_overload (fn); + release_fn.stash_name ("release"); + + // release matrix + fn = create_external (JIT_FN (octave_jit_release_matrix), 0, matrix); + release_fn.add_overload (fn); + + // destroy + destroy_fn = release_fn; + destroy_fn.stash_name ("destroy"); + destroy_fn.add_overload (create_identity(scalar)); + destroy_fn.add_overload (create_identity(boolean)); + destroy_fn.add_overload (create_identity(index)); + destroy_fn.add_overload (create_identity(complex)); + + // -------------------- scalar related operations -------------------- + + // now for binary scalar operations + add_binary_op (scalar, octave_value::op_add, llvm::Instruction::FAdd); + add_binary_op (scalar, octave_value::op_sub, llvm::Instruction::FSub); + add_binary_op (scalar, octave_value::op_mul, llvm::Instruction::FMul); + add_binary_op (scalar, octave_value::op_el_mul, llvm::Instruction::FMul); + + add_binary_fcmp (scalar, octave_value::op_lt, llvm::CmpInst::FCMP_ULT); + add_binary_fcmp (scalar, octave_value::op_le, llvm::CmpInst::FCMP_ULE); + add_binary_fcmp (scalar, octave_value::op_eq, llvm::CmpInst::FCMP_UEQ); + add_binary_fcmp (scalar, octave_value::op_ge, llvm::CmpInst::FCMP_UGE); + add_binary_fcmp (scalar, octave_value::op_gt, llvm::CmpInst::FCMP_UGT); + add_binary_fcmp (scalar, octave_value::op_ne, llvm::CmpInst::FCMP_UNE); + + jit_function gripe_div0 = create_external (JIT_FN (warn_divide_by_zero), 0); + gripe_div0.mark_can_error (); + + // divide is annoying because it might error + fn = create_internal ("octave_jit_div_scalar_scalar", scalar, scalar, scalar); + fn.mark_can_error (); + + llvm::BasicBlock *body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::BasicBlock *warn_block = fn.new_block ("warn"); + llvm::BasicBlock *normal_block = fn.new_block ("normal"); + + llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); + llvm::Value *check = builder.CreateFCmpUEQ (zero, fn.argument (builder, 1)); + builder.CreateCondBr (check, warn_block, normal_block); + + builder.SetInsertPoint (warn_block); + gripe_div0.call (builder); + builder.CreateBr (normal_block); + + builder.SetInsertPoint (normal_block); + llvm::Value *ret = builder.CreateFDiv (fn.argument (builder, 0), + fn.argument (builder, 1)); + fn.do_return (builder, ret); + } + binary_ops[octave_value::op_div].add_overload (fn); + binary_ops[octave_value::op_el_div].add_overload (fn); + + // ldiv is the same as div with the operators reversed + fn = mirror_binary (fn); + binary_ops[octave_value::op_ldiv].add_overload (fn); + binary_ops[octave_value::op_el_ldiv].add_overload (fn); + + // In general, the result of scalar ^ scalar is a complex number. We might + // be able to improve on this if we keep track of the range of values + // variables can take on. + fn = create_external (JIT_FN (octave_jit_pow_scalar_scalar), complex, scalar, + scalar); + binary_ops[octave_value::op_pow].add_overload (fn); + binary_ops[octave_value::op_el_pow].add_overload (fn); + + // now for unary scalar operations + // FIXME: Impelment not + fn = create_internal ("octave_jit_++", scalar, scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *one = llvm::ConstantFP::get (scalar_t, 1); + llvm::Value *val = fn.argument (builder, 0); + val = builder.CreateFAdd (val, one); + fn.do_return (builder, val); + } + unary_ops[octave_value::op_incr].add_overload (fn); + + fn = create_internal ("octave_jit_--", scalar, scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *one = llvm::ConstantFP::get (scalar_t, 1); + llvm::Value *val = fn.argument (builder, 0); + val = builder.CreateFSub (val, one); + fn.do_return (builder, val); + } + unary_ops[octave_value::op_decr].add_overload (fn); + + fn = create_internal ("octave_jit_uminus", scalar, scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *mone = llvm::ConstantFP::get (scalar_t, -1); + llvm::Value *val = fn.argument (builder, 0); + val = builder.CreateFMul (val, mone); + fn.do_return (builder, val); + } + unary_ops[octave_value::op_uminus].add_overload (fn); + + fn = create_identity (scalar); + unary_ops[octave_value::op_uplus].add_overload (fn); + unary_ops[octave_value::op_transpose].add_overload (fn); + unary_ops[octave_value::op_hermitian].add_overload (fn); + + // now for binary complex operations + fn = create_internal ("octave_jit_+_complex_complex", complex, complex, + complex); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + llvm::Value *real = builder.CreateFAdd (complex_real (lhs), + complex_real (rhs)); + llvm::Value *imag = builder.CreateFAdd (complex_imag (lhs), + complex_imag (rhs)); + fn.do_return (builder, complex_new (real, imag)); + } + binary_ops[octave_value::op_add].add_overload (fn); + + fn = create_internal ("octave_jit_-_complex_complex", complex, complex, + complex); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + llvm::Value *real = builder.CreateFSub (complex_real (lhs), + complex_real (rhs)); + llvm::Value *imag = builder.CreateFSub (complex_imag (lhs), + complex_imag (rhs)); + fn.do_return (builder, complex_new (real, imag)); + } + binary_ops[octave_value::op_sub].add_overload (fn); + + fn = create_external (JIT_FN (octave_jit_complex_mul), + complex, complex, complex); + binary_ops[octave_value::op_mul].add_overload (fn); + binary_ops[octave_value::op_el_mul].add_overload (fn); + + jit_function complex_div = create_external (JIT_FN (octave_jit_complex_div), + complex, complex, complex); + complex_div.mark_can_error (); + binary_ops[octave_value::op_div].add_overload (fn); + binary_ops[octave_value::op_ldiv].add_overload (fn); + + fn = create_external (JIT_FN (octave_jit_pow_complex_complex), complex, + complex, complex); + binary_ops[octave_value::op_pow].add_overload (fn); + binary_ops[octave_value::op_el_pow].add_overload (fn); + + fn = create_internal ("octave_jit_*_scalar_complex", complex, scalar, + complex); + jit_function mul_scalar_complex = fn; + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::BasicBlock *complex_mul = fn.new_block ("complex_mul"); + llvm::BasicBlock *scalar_mul = fn.new_block ("scalar_mul"); + + llvm::Value *fzero = llvm::ConstantFP::get (scalar_t, 0); + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + + llvm::Value *cmp = builder.CreateFCmpUEQ (complex_imag (rhs), fzero); + builder.CreateCondBr (cmp, scalar_mul, complex_mul); + + builder.SetInsertPoint (scalar_mul); + llvm::Value *temp = complex_real (rhs); + temp = builder.CreateFMul (lhs, temp); + fn.do_return (builder, complex_new (temp, fzero), false); + + builder.SetInsertPoint (complex_mul); + temp = complex_new (builder.CreateFMul (lhs, complex_real (rhs)), + builder.CreateFMul (lhs, complex_imag (rhs))); + fn.do_return (builder, temp); + } + binary_ops[octave_value::op_mul].add_overload (fn); + binary_ops[octave_value::op_el_mul].add_overload (fn); + + fn = mirror_binary (mul_scalar_complex); + binary_ops[octave_value::op_mul].add_overload (fn); + binary_ops[octave_value::op_el_mul].add_overload (fn); + + fn = create_internal ("octave_jit_+_scalar_complex", complex, scalar, + complex); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + llvm::Value *real = builder.CreateFAdd (lhs, complex_real (rhs)); + fn.do_return (builder, complex_real (rhs, real)); + } + binary_ops[octave_value::op_add].add_overload (fn); + + fn = mirror_binary (fn); + binary_ops[octave_value::op_add].add_overload (fn); + + fn = create_internal ("octave_jit_-_complex_scalar", complex, complex, + scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + llvm::Value *real = builder.CreateFSub (complex_real (lhs), rhs); + fn.do_return (builder, complex_real (lhs, real)); + } + binary_ops[octave_value::op_sub].add_overload (fn); + + fn = create_internal ("octave_jit_-_scalar_complex", complex, scalar, + complex); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *lhs = fn.argument (builder, 0); + llvm::Value *rhs = fn.argument (builder, 1); + llvm::Value *real = builder.CreateFSub (lhs, complex_real (rhs)); + fn.do_return (builder, complex_real (rhs, real)); + } + binary_ops[octave_value::op_sub].add_overload (fn); + + fn = create_external (JIT_FN (octave_jit_pow_scalar_complex), complex, scalar, + complex); + binary_ops[octave_value::op_pow].add_overload (fn); + binary_ops[octave_value::op_el_pow].add_overload (fn); + + fn = create_external (JIT_FN (octave_jit_pow_complex_scalar), complex, + complex, scalar); + binary_ops[octave_value::op_pow].add_overload (fn); + binary_ops[octave_value::op_el_pow].add_overload (fn); + + // now for binary index operators + add_binary_op (index, octave_value::op_add, llvm::Instruction::Add); + + // and binary bool operators + add_binary_op (boolean, octave_value::op_el_or, llvm::Instruction::Or); + add_binary_op (boolean, octave_value::op_el_and, llvm::Instruction::And); + + // now for printing functions + print_fn.stash_name ("print"); + add_print (any, reinterpret_cast<void *> (&octave_jit_print_any)); + add_print (scalar, reinterpret_cast<void *> (&octave_jit_print_scalar)); + + // initialize for loop + for_init_fn.stash_name ("for_init"); + + fn = create_internal ("octave_jit_for_range_init", index, range); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *zero = llvm::ConstantInt::get (index_t, 0); + fn.do_return (builder, zero); + } + for_init_fn.add_overload (fn); + + // bounds check for for loop + for_check_fn.stash_name ("for_check"); + + fn = create_internal ("octave_jit_for_range_check", boolean, range, index); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *nelem + = builder.CreateExtractValue (fn.argument (builder, 0), 3); + llvm::Value *idx = fn.argument (builder, 1); + llvm::Value *ret = builder.CreateICmpULT (idx, nelem); + fn.do_return (builder, ret); + } + for_check_fn.add_overload (fn); + + // index variabe for for loop + for_index_fn.stash_name ("for_index"); + + fn = create_internal ("octave_jit_for_range_idx", scalar, range, index); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *idx = fn.argument (builder, 1); + llvm::Value *didx = builder.CreateSIToFP (idx, scalar_t); + llvm::Value *rng = fn.argument (builder, 0); + llvm::Value *base = builder.CreateExtractValue (rng, 0); + llvm::Value *inc = builder.CreateExtractValue (rng, 2); + + llvm::Value *ret = builder.CreateFMul (didx, inc); + ret = builder.CreateFAdd (base, ret); + fn.do_return (builder, ret); + } + for_index_fn.add_overload (fn); + + // logically true + logically_true_fn.stash_name ("logically_true"); + + jit_function gripe_nantl + = create_external (JIT_FN (octave_jit_octave::err_nan_to_logical_conversion), 0); + gripe_nantl.mark_can_error (); + + fn = create_internal ("octave_jit_logically_true_scalar", boolean, scalar); + fn.mark_can_error (); + + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::BasicBlock *error_block = fn.new_block ("error"); + llvm::BasicBlock *normal_block = fn.new_block ("normal"); + + llvm::Value *check = builder.CreateFCmpUNE (fn.argument (builder, 0), + fn.argument (builder, 0)); + builder.CreateCondBr (check, error_block, normal_block); + + builder.SetInsertPoint (error_block); + gripe_nantl.call (builder); + builder.CreateBr (normal_block); + builder.SetInsertPoint (normal_block); + + llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); + llvm::Value *ret = builder.CreateFCmpONE (fn.argument (builder, 0), zero); + fn.do_return (builder, ret); + } + logically_true_fn.add_overload (fn); + + // logically_true boolean + fn = create_identity (boolean); + logically_true_fn.add_overload (fn); + + // make_range + // FIXME: May be benificial to implement all in LLVM + make_range_fn.stash_name ("make_range"); + jit_function compute_nelem + = create_external (JIT_FN (octave_jit_compute_nelem), + index, scalar, scalar, scalar); + + fn = create_internal ("octave_jit_make_range", range, scalar, scalar, scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *base = fn.argument (builder, 0); + llvm::Value *limit = fn.argument (builder, 1); + llvm::Value *inc = fn.argument (builder, 2); + llvm::Value *nelem = compute_nelem.call (builder, base, limit, inc); + + llvm::Value *dzero = llvm::ConstantFP::get (scalar_t, 0); + llvm::Value *izero = llvm::ConstantInt::get (index_t, 0); + llvm::Value *rng = llvm::ConstantStruct::get (range_t, dzero, dzero, dzero, + izero, NULL); + rng = builder.CreateInsertValue (rng, base, 0); + rng = builder.CreateInsertValue (rng, limit, 1); + rng = builder.CreateInsertValue (rng, inc, 2); + rng = builder.CreateInsertValue (rng, nelem, 3); + fn.do_return (builder, rng); + } + make_range_fn.add_overload (fn); + + // paren_subsref + jit_type *jit_int = intN (sizeof (int) * 8); + llvm::Type *int_t = jit_int->to_llvm (); + jit_function ginvalid_index + = create_external (JIT_FN (octave_jit_ginvalid_index), 0); + jit_function gindex_range = create_external (JIT_FN (octave_jit_gindex_range), + 0, jit_int, jit_int, index, + index); + + fn = create_internal ("()subsref", scalar, matrix, scalar); + fn.mark_can_error (); + + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *one_idx = llvm::ConstantInt::get (index_t, 1); + llvm::Value *one_int = llvm::ConstantInt::get (int_t, 1); + + llvm::Value *undef = llvm::UndefValue::get (scalar_t); + llvm::Value *mat = fn.argument (builder, 0); + llvm::Value *idx = fn.argument (builder, 1); + + // convert index to scalar to integer, and check index >= 1 + llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); + llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); + llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); + llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one_idx); + llvm::Value *cond = builder.CreateOr (cond0, cond1); + + llvm::BasicBlock *done = fn.new_block ("done"); + llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); + llvm::BasicBlock *normal = fn.new_block ("normal", done); + builder.CreateCondBr (cond, conv_error, normal); + + builder.SetInsertPoint (conv_error); + ginvalid_index.call (builder); + builder.CreateBr (done); + + builder.SetInsertPoint (normal); + llvm::Value *len + = builder.CreateExtractValue (mat, llvm::ArrayRef<unsigned> (2)); + cond = builder.CreateICmpSGT (int_idx, len); + + llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); + llvm::BasicBlock *success = fn.new_block ("success", done); + builder.CreateCondBr (cond, bounds_error, success); + + builder.SetInsertPoint (bounds_error); + gindex_range.call (builder, one_int, one_int, int_idx, len); + builder.CreateBr (done); + + builder.SetInsertPoint (success); + llvm::Value *data = builder.CreateExtractValue (mat, + llvm::ArrayRef<unsigned> (1)); + llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); + llvm::Value *ret = builder.CreateLoad (gep); + builder.CreateBr (done); + + builder.SetInsertPoint (done); + + llvm::PHINode *merge = llvm::PHINode::Create (scalar_t, 3); + builder.Insert (merge); + merge->addIncoming (undef, conv_error); + merge->addIncoming (undef, bounds_error); + merge->addIncoming (ret, success); + fn.do_return (builder, merge); + } + paren_subsref_fn.add_overload (fn); + + // paren subsasgn + paren_subsasgn_fn.stash_name ("()subsasgn"); + + jit_function resize_paren_subsasgn + = create_external (JIT_FN (octave_jit_paren_subsasgn_impl), matrix, matrix, + index, scalar); + + fn = create_internal ("octave_jit_paren_subsasgn", matrix, matrix, scalar, + scalar); + fn.mark_can_error (); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *one_idx = llvm::ConstantInt::get (index_t, 1); + llvm::Value *one_int = llvm::ConstantInt::get (int_t, 1); + + llvm::Value *mat = fn.argument (builder, 0); + llvm::Value *idx = fn.argument (builder, 1); + llvm::Value *value = fn.argument (builder, 2); + + llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); + llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); + llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); + llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one_idx); + llvm::Value *cond = builder.CreateOr (cond0, cond1); + + llvm::BasicBlock *done = fn.new_block ("done"); + + llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); + llvm::BasicBlock *normal = fn.new_block ("normal", done); + builder.CreateCondBr (cond, conv_error, normal); + builder.SetInsertPoint (conv_error); + ginvalid_index.call (builder); + builder.CreateBr (done); + + builder.SetInsertPoint (normal); + llvm::Value *len = builder.CreateExtractValue (mat, 2); + cond0 = builder.CreateICmpSGT (int_idx, len); + + llvm::Value *rcount = builder.CreateExtractValue (mat, 0); + rcount = builder.CreateLoad (rcount); + cond1 = builder.CreateICmpSGT (rcount, one_int); + cond = builder.CreateOr (cond0, cond1); + + llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); + llvm::BasicBlock *success = fn.new_block ("success", done); + builder.CreateCondBr (cond, bounds_error, success); + + // resize on out of bounds access + builder.SetInsertPoint (bounds_error); + llvm::Value *resize_result = resize_paren_subsasgn.call (builder, mat, + int_idx, value); + builder.CreateBr (done); + + builder.SetInsertPoint (success); + llvm::Value *data + = builder.CreateExtractValue (mat, llvm::ArrayRef<unsigned> (1)); + llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); + builder.CreateStore (value, gep); + builder.CreateBr (done); + + builder.SetInsertPoint (done); + + llvm::PHINode *merge = llvm::PHINode::Create (matrix_t, 3); + builder.Insert (merge); + merge->addIncoming (mat, conv_error); + merge->addIncoming (resize_result, bounds_error); + merge->addIncoming (mat, success); + fn.do_return (builder, merge); + } + paren_subsasgn_fn.add_overload (fn); + + fn = create_external (JIT_FN (octave_jit_paren_subsasgn_matrix_range), matrix, + matrix, range, scalar); + fn.mark_can_error (); + paren_subsasgn_fn.add_overload (fn); + + end1_fn.stash_name ("end1"); + fn = create_internal ("octave_jit_end1_matrix", scalar, matrix, index, index); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *mat = fn.argument (builder, 0); + llvm::Value *ret = builder.CreateExtractValue (mat, 2); + fn.do_return (builder, builder.CreateSIToFP (ret, scalar_t)); + } + end1_fn.add_overload (fn); + + end_fn.stash_name ("end"); + fn = create_external (JIT_FN (octave_jit_end_matrix),scalar, matrix, index, + index); + end_fn.add_overload (fn); + + // -------------------- create_undef -------------------- + create_undef_fn.stash_name ("create_undef"); + fn = create_external (JIT_FN (octave_jit_create_undef), any); + create_undef_fn.add_overload (fn); + + casts[any->type_id ()].stash_name ("(any)"); + casts[scalar->type_id ()].stash_name ("(scalar)"); + casts[complex->type_id ()].stash_name ("(complex)"); + casts[matrix->type_id ()].stash_name ("(matrix)"); + casts[range->type_id ()].stash_name ("(range)"); + + // cast any <- matrix + fn = create_external (JIT_FN (octave_jit_cast_any_matrix), any, matrix); + casts[any->type_id ()].add_overload (fn); + + // cast matrix <- any + fn = create_external (JIT_FN (octave_jit_cast_matrix_any), matrix, any); + casts[matrix->type_id ()].add_overload (fn); + + // cast any <- range + fn = create_external (JIT_FN (octave_jit_cast_any_range), any, range); + casts[any->type_id ()].add_overload (fn); + + // cast range <- any + fn = create_external (JIT_FN (octave_jit_cast_range_any), range, any); + casts[range->type_id ()].add_overload (fn); + + // cast any <- scalar + fn = create_external (JIT_FN (octave_jit_cast_any_scalar), any, scalar); + casts[any->type_id ()].add_overload (fn); + + // cast scalar <- any + fn = create_external (JIT_FN (octave_jit_cast_scalar_any), scalar, any); + casts[scalar->type_id ()].add_overload (fn); + + // cast any <- complex + fn = create_external (JIT_FN (octave_jit_cast_any_complex), any, complex); + casts[any->type_id ()].add_overload (fn); + + // cast complex <- any + fn = create_external (JIT_FN (octave_jit_cast_complex_any), complex, any); + casts[complex->type_id ()].add_overload (fn); + + // cast complex <- scalar + fn = create_internal ("octave_jit_cast_complex_scalar", complex, scalar); + body = fn.new_block (); + builder.SetInsertPoint (body); + { + llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); + fn.do_return (builder, complex_new (fn.argument (builder, 0), zero)); + } + casts[complex->type_id ()].add_overload (fn); + + // cast scalar <- complex + fn = create_internal ("octave_jit_cast_scalar_complex", scalar, complex); + body = fn.new_block (); + builder.SetInsertPoint (body); + fn.do_return (builder, complex_real (fn.argument (builder, 0))); + casts[scalar->type_id ()].add_overload (fn); + + // cast any <- any + fn = create_identity (any); + casts[any->type_id ()].add_overload (fn); + + // cast scalar <- scalar + fn = create_identity (scalar); + casts[scalar->type_id ()].add_overload (fn); + + // cast complex <- complex + fn = create_identity (complex); + casts[complex->type_id ()].add_overload (fn); + + // -------------------- builtin functions -------------------- + add_builtin ("#unknown_function"); + unknown_function = builtins["#unknown_function"]; + + add_builtin ("sin"); + register_intrinsic ("sin", llvm::Intrinsic::sin, scalar, scalar); + register_generic ("sin", matrix, matrix); + + add_builtin ("cos"); + register_intrinsic ("cos", llvm::Intrinsic::cos, scalar, scalar); + register_generic ("cos", matrix, matrix); + + add_builtin ("exp"); + register_intrinsic ("exp", llvm::Intrinsic::exp, scalar, scalar); + register_generic ("exp", matrix, matrix); + + add_builtin ("balance"); + register_generic ("balance", matrix, matrix); + + add_builtin ("cond"); + register_generic ("cond", scalar, matrix); + + add_builtin ("det"); + register_generic ("det", scalar, matrix); + + add_builtin ("norm"); + register_generic ("norm", scalar, matrix); + + add_builtin ("rand"); + register_generic ("rand", matrix, scalar); + register_generic ("rand", matrix, std::vector<jit_type *> (2, scalar)); + + add_builtin ("magic"); + register_generic ("magic", matrix, scalar); + register_generic ("magic", matrix, std::vector<jit_type *> (2, scalar)); + + add_builtin ("eye"); + register_generic ("eye", matrix, scalar); + register_generic ("eye", matrix, std::vector<jit_type *> (2, scalar)); + + add_builtin ("mod"); + register_generic ("mod", scalar, std::vector<jit_type *> (2, scalar)); + + casts.resize (next_id + 1); + jit_function any_id = create_identity (any); + jit_function grab_any = create_external (JIT_FN (octave_jit_grab_any), + any, any); + jit_function release_any = get_release (any); + std::vector<jit_type *> args; + args.resize (1); + + for (std::map<std::string, jit_type *>::iterator iter = builtins.begin (); + iter != builtins.end (); ++iter) + { + jit_type *btype = iter->second; + args[0] = btype; + + grab_fn.add_overload (jit_function (grab_any, btype, args)); + release_fn.add_overload (jit_function (release_any, 0, args)); + casts[any->type_id ()].add_overload (jit_function (any_id, any, args)); + + args[0] = any; + casts[btype->type_id ()].add_overload (jit_function (any_id, btype, + args)); + } +} + +const jit_function& +jit_typeinfo::do_end (jit_value *value, jit_value *idx, jit_value *count) +{ + jit_const_index *ccount = dynamic_cast<jit_const_index *> (count); + if (ccount && ccount->value () == 1) + return end1_fn.overload (value->type (), idx->type (), count->type ()); + + return end_fn.overload (value->type (), idx->type (), count->type ()); +} + +jit_type* +jit_typeinfo::new_type (const std::string& name, jit_type *parent, + llvm::Type *llvm_type, bool skip_paren) +{ + jit_type *ret = new jit_type (name, parent, llvm_type, skip_paren, next_id++); + id_to_type.push_back (ret); + return ret; +} + +void +jit_typeinfo::add_print (jit_type *ty, void *fptr) +{ + std::stringstream name; + name << "octave_jit_print_" << ty->name (); + jit_function fn = create_external (engine, fptr, name.str (), + 0, intN (8), ty); + print_fn.add_overload (fn); +} + +// FIXME: cp between add_binary_op, add_binary_icmp, and add_binary_fcmp +void +jit_typeinfo::add_binary_op (jit_type *ty, int op, int llvm_op) +{ + std::stringstream fname; + octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); + fname << "octave_jit_" << octave_value::binary_op_as_string (ov_op) + << "_" << ty->name (); + + jit_function fn = create_internal (fname.str (), ty, ty, ty); + llvm::BasicBlock *block = fn.new_block (); + builder.SetInsertPoint (block); + llvm::Instruction::BinaryOps temp + = static_cast<llvm::Instruction::BinaryOps>(llvm_op); + + llvm::Value *ret = builder.CreateBinOp (temp, fn.argument (builder, 0), + fn.argument (builder, 1)); + fn.do_return (builder, ret); + binary_ops[op].add_overload (fn); +} + +void +jit_typeinfo::add_binary_icmp (jit_type *ty, int op, int llvm_op) +{ + std::stringstream fname; + octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); + fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) + << "_" << ty->name (); + + jit_function fn = create_internal (fname.str (), boolean, ty, ty); + llvm::BasicBlock *block = fn.new_block (); + builder.SetInsertPoint (block); + llvm::CmpInst::Predicate temp + = static_cast<llvm::CmpInst::Predicate>(llvm_op); + llvm::Value *ret = builder.CreateICmp (temp, fn.argument (builder, 0), + fn.argument (builder, 1)); + fn.do_return (builder, ret); + binary_ops[op].add_overload (fn); +} + +void +jit_typeinfo::add_binary_fcmp (jit_type *ty, int op, int llvm_op) +{ + std::stringstream fname; + octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); + fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) + << "_" << ty->name (); + + jit_function fn = create_internal (fname.str (), boolean, ty, ty); + llvm::BasicBlock *block = fn.new_block (); + builder.SetInsertPoint (block); + llvm::CmpInst::Predicate temp + = static_cast<llvm::CmpInst::Predicate>(llvm_op); + llvm::Value *ret = builder.CreateFCmp (temp, fn.argument (builder, 0), + fn.argument (builder, 1)); + fn.do_return (builder, ret); + binary_ops[op].add_overload (fn); +} + +jit_function +jit_typeinfo::create_function (jit_convention::type cc, const llvm::Twine& name, + jit_type *ret, + const std::vector<jit_type *>& args) +{ + jit_function result (module, cc, name, ret, args); + return result; +} + +jit_function +jit_typeinfo::create_identity (jit_type *type) +{ + size_t id = type->type_id (); + if (id >= identities.size ()) + identities.resize (id + 1); + + if (! identities[id].valid ()) + { + std::stringstream name; + name << "id_" << type->name (); + + jit_function fn = create_internal (name.str (), type, type); + llvm::BasicBlock *body = fn.new_block (); + builder.SetInsertPoint (body); + fn.do_return (builder, fn.argument (builder, 0)); + return identities[id] = fn; + } + + return identities[id]; +} + +llvm::Value * +jit_typeinfo::do_insert_error_check (llvm::IRBuilderD& abuilder) +{ + return abuilder.CreateLoad (lerror_state); +} + +llvm::Value * +jit_typeinfo::do_insert_interrupt_check (llvm::IRBuilderD& abuilder) +{ + llvm::LoadInst *val = abuilder.CreateLoad (loctave_interrupt_state); + val->setVolatile (true); + return abuilder.CreateICmpSGT (val, abuilder.getInt32 (0)); +} + +void +jit_typeinfo::add_builtin (const std::string& name) +{ + jit_type *btype = new_type (name, any, any->to_llvm (), true); + builtins[name] = btype; + + octave_builtin *ov_builtin = find_builtin (name); + if (ov_builtin) + ov_builtin->stash_jit (*btype); +} + +void +jit_typeinfo::register_intrinsic (const std::string& name, size_t iid, + jit_type *result, + const std::vector<jit_type *>& args) +{ + jit_type *builtin_type = builtins[name]; + size_t nargs = args.size (); + llvm::SmallVector<llvm::Type *, 5> llvm_args (nargs); + for (size_t i = 0; i < nargs; ++i) + llvm_args[i] = args[i]->to_llvm (); + + llvm::Intrinsic::ID id = static_cast<llvm::Intrinsic::ID> (iid); + llvm::Function *ifun = llvm::Intrinsic::getDeclaration (module, id, + llvm_args); + std::stringstream fn_name; + fn_name << "octave_jit_" << name; + + std::vector<jit_type *> args1 (nargs + 1); + args1[0] = builtin_type; + std::copy (args.begin (), args.end (), args1.begin () + 1); + + // The first argument will be the Octave function, but we already know that + // the function call is the equivalent of the intrinsic, so we ignore it and + // call the intrinsic with the remaining arguments. + jit_function fn = create_internal (fn_name.str (), result, args1); + llvm::BasicBlock *body = fn.new_block (); + builder.SetInsertPoint (body); + + llvm::SmallVector<llvm::Value *, 5> fargs (nargs); + for (size_t i = 0; i < nargs; ++i) + fargs[i] = fn.argument (builder, i + 1); + + llvm::Value *ret = builder.CreateCall (ifun, fargs); + fn.do_return (builder, ret); + paren_subsref_fn.add_overload (fn); +} + +octave_builtin * +jit_typeinfo::find_builtin (const std::string& name) +{ + // FIXME: Finalize what we want to store in octave_builtin, then add functions + // to access these values in octave_value + octave_value ov_builtin = symbol_table::find (name); + return dynamic_cast<octave_builtin *> (ov_builtin.internal_rep ()); +} + +void +jit_typeinfo::register_generic (const std::string& name, jit_type *result, + const std::vector<jit_type *>& args) +{ + octave_builtin *builtin = find_builtin (name); + if (! builtin) + return; + + std::vector<jit_type *> fn_args (args.size () + 1); + fn_args[0] = builtins[name]; + std::copy (args.begin (), args.end (), fn_args.begin () + 1); + jit_function fn = create_internal (name, result, fn_args); + fn.mark_can_error (); + llvm::BasicBlock *block = fn.new_block (); + builder.SetInsertPoint (block); + llvm::Type *any_t = any->to_llvm (); + llvm::ArrayType *array_t = llvm::ArrayType::get (any_t, args.size ()); + llvm::Value *array = llvm::UndefValue::get (array_t); + for (size_t i = 0; i < args.size (); ++i) + { + llvm::Value *arg = fn.argument (builder, i + 1); + jit_function agrab = get_grab (args[i]); + if (agrab.valid ()) + arg = agrab.call (builder, arg); + jit_function acast = cast (any, args[i]); + array = builder.CreateInsertValue (array, acast.call (builder, arg), i); + } + + llvm::Value *array_mem = builder.CreateAlloca (array_t); + builder.CreateStore (array, array_mem); + array = builder.CreateBitCast (array_mem, any_t->getPointerTo ()); + + jit_type *jintTy = intN (sizeof (octave_builtin::fcn) * 8); + llvm::Type *intTy = jintTy->to_llvm (); + size_t fcn_int = reinterpret_cast<size_t> (builtin->function ()); + llvm::Value *fcn = llvm::ConstantInt::get (intTy, fcn_int); + llvm::Value *nargin = llvm::ConstantInt::get (intTy, args.size ()); + size_t result_int = reinterpret_cast<size_t> (result); + llvm::Value *res_llvm = llvm::ConstantInt::get (intTy, result_int); + llvm::Value *ret = any_call.call (builder, fcn, nargin, array, res_llvm); + + jit_function cast_result = cast (result, any); + fn.do_return (builder, cast_result.call (builder, ret)); + paren_subsref_fn.add_overload (fn); +} + +jit_function +jit_typeinfo::mirror_binary (const jit_function& fn) +{ + jit_function ret = create_internal (fn.name () + "_reverse", + fn.result (), fn.argument_type (1), + fn.argument_type (0)); + if (fn.can_error ()) + ret.mark_can_error (); + + llvm::BasicBlock *body = ret.new_block (); + builder.SetInsertPoint (body); + llvm::Value *result = fn.call (builder, ret.argument (builder, 1), + ret.argument (builder, 0)); + if (ret.result ()) + ret.do_return (builder, result); + else + ret.do_return (builder); + + return ret; +} + +llvm::Value * +jit_typeinfo::pack_complex (llvm::IRBuilderD& bld, llvm::Value *cplx) +{ + llvm::Type *complex_ret = instance->complex_ret; + llvm::Value *real = bld.CreateExtractValue (cplx, 0); + llvm::Value *imag = bld.CreateExtractValue (cplx, 1); + llvm::Value *ret = llvm::UndefValue::get (complex_ret); + + unsigned int re_idx[] = {0, 0}; + unsigned int im_idx[] = {0, 1}; + ret = bld.CreateInsertValue (ret, real, re_idx); + return bld.CreateInsertValue (ret, imag, im_idx); +} + +llvm::Value * +jit_typeinfo::unpack_complex (llvm::IRBuilderD& bld, llvm::Value *result) +{ + unsigned int re_idx[] = {0, 0}; + unsigned int im_idx[] = {0, 1}; + + llvm::Type *complex_t = get_complex ()->to_llvm (); + llvm::Value *real = bld.CreateExtractValue (result, re_idx); + llvm::Value *imag = bld.CreateExtractValue (result, im_idx); + llvm::Value *ret = llvm::UndefValue::get (complex_t); + + ret = bld.CreateInsertValue (ret, real, 0); + return bld.CreateInsertValue (ret, imag, 1); +} + +llvm::Value * +jit_typeinfo::complex_real (llvm::Value *cx) +{ + return builder.CreateExtractValue (cx, 0); +} + +llvm::Value * +jit_typeinfo::complex_real (llvm::Value *cx, llvm::Value *real) +{ + return builder.CreateInsertValue (cx, real, 0); +} + +llvm::Value * +jit_typeinfo::complex_imag (llvm::Value *cx) +{ + return builder.CreateExtractValue (cx, 1); +} + +llvm::Value * +jit_typeinfo::complex_imag (llvm::Value *cx, llvm::Value *imag) +{ + return builder.CreateInsertValue (cx, imag, 1); +} + +llvm::Value * +jit_typeinfo::complex_new (llvm::Value *real, llvm::Value *imag) +{ + llvm::Value *ret = llvm::UndefValue::get (complex->to_llvm ()); + ret = complex_real (ret, real); + return complex_imag (ret, imag); +} + +void +jit_typeinfo::create_int (size_t nbits) +{ + std::stringstream tname; + tname << "int" << nbits; + ints[nbits] = new_type (tname.str (), any, llvm::Type::getIntNTy (context, + nbits)); +} + +jit_type * +jit_typeinfo::intN (size_t nbits) const +{ + std::map<size_t, jit_type *>::const_iterator iter = ints.find (nbits); + if (iter != ints.end ()) + return iter->second; + + throw jit_fail_exception ("No such integer type"); +} + +jit_type * +jit_typeinfo::do_type_of (const octave_value &ov) const +{ + if (ov.is_function ()) + { + // FIXME: This is ugly, we need to finalize how we want to do this, then + // have octave_value fully support the needed functionality + octave_builtin *builtin + = dynamic_cast<octave_builtin *> (ov.internal_rep ()); + return builtin && builtin->to_jit () ? builtin->to_jit () + : unknown_function; + } + + if (ov.is_range ()) + return get_range (); + + if (ov.is_double_type () && ! ov.is_complex_type ()) + { + if (ov.is_real_scalar ()) + return get_scalar (); + + if (ov.is_matrix_type ()) + return get_matrix (); + } + + if (ov.is_complex_scalar ()) + { + Complex cv = ov.complex_value (); + + // We don't really represent complex values, instead we represent + // complex_or_scalar. If the imag value is zero, we assume a scalar. + if (cv.imag () != 0) + return get_complex (); + } + + return get_any (); +} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-typeinfo.h Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,851 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +#if ! defined (octave_jit_typeinfo_h) +#define octave_jit_typeinfo_h 1 + +#include "octave-config.h" + +#if defined (HAVE_LLVM) + +#include <map> +#include <vector> + +#include "Range.h" +#include "jit-util.h" + +// Defines the type system used by jit and a singleton class, jit_typeinfo, to +// manage the types. +// +// FIXME: +// Operations are defined and implemented in jit_typeinfo. Eventually they +// should be moved elsewhere. (just like with octave_typeinfo) + +// jit_range is compatible with the llvm range structure +struct +jit_range +{ + jit_range (const Range& from) : base (from.base ()), limit (from.limit ()), + inc (from.inc ()), nelem (from.numel ()) + { } + + operator Range () const + { + return Range (base, limit, inc); + } + + bool all_elements_are_ints () const; + + double base; + double limit; + double inc; + octave_idx_type nelem; +}; + +std::ostream& operator << (std::ostream& os, const jit_range& rng); + +// jit_array is compatible with the llvm array/matrix structures +template <typename T, typename U> +struct +jit_array +{ + jit_array () : array (0) { } + + jit_array (T& from) : array (new T (from)) + { + update (); + } + + void update (void) + { + ref_count = array->jit_ref_count (); + slice_data = array->jit_slice_data () - 1; + slice_len = array->numel (); + dimensions = array->jit_dimensions (); + } + + void update (T *aarray) + { + array = aarray; + update (); + } + + operator T () const + { + return *array; + } + + int *ref_count; + + U *slice_data; + octave_idx_type slice_len; + octave_idx_type *dimensions; + + T *array; +}; + +typedef jit_array<NDArray, double> jit_matrix; + +std::ostream& operator << (std::ostream& os, const jit_matrix& mat); + +// calling convention +namespace jit_convention +{ + enum + type + { + // internal to jit + internal, + + // an external C call + external, + + length + }; +} + +// Used to keep track of estimated (infered) types during JIT. This is a +// hierarchical type system which includes both concrete and abstract types. +// +// The types form a lattice. Currently we only allow for one parent type, but +// eventually we may allow for multiple predecessors. +class +jit_type +{ +public: + typedef llvm::Value *(*convert_fn) (llvm::IRBuilderD&, llvm::Value *); + + jit_type (const std::string& aname, jit_type *aparent, llvm::Type *allvm_type, + bool askip_paren, int aid); + + // a user readable type name + const std::string& name (void) const { return mname; } + + // a unique id for the type + int type_id (void) const { return mid; } + + // An abstract base type, may be null + jit_type *parent (void) const { return mparent; } + + // convert to an llvm type + llvm::Type *to_llvm (void) const { return llvm_type; } + + // how this type gets passed as a function argument + llvm::Type *to_llvm_arg (void) const; + + size_t depth (void) const { return mdepth; } + + bool skip_paren (void) const { return mskip_paren; } + + // -------------------- Calling Convention information -------------------- + + // A function declared like: mytype foo (int arg0, int arg1); + // Will be converted to: void foo (mytype *retval, int arg0, int arg1) + // if mytype is sret. The caller is responsible for allocating space for + // retval. (on the stack) + bool sret (jit_convention::type cc) const { return msret[cc]; } + + void mark_sret (jit_convention::type cc) + { msret[cc] = true; } + + // A function like: void foo (mytype arg0) + // Will be converted to: void foo (mytype *arg0) + // Basically just pass by reference. + bool pointer_arg (jit_convention::type cc) const { return mpointer_arg[cc]; } + + void mark_pointer_arg (jit_convention::type cc) + { mpointer_arg[cc] = true; } + + // Convert into an equivalent form before calling. For example, complex is + // represented as two values llvm vector, but we need to pass it as a two + // valued llvm structure to C functions. + convert_fn pack (jit_convention::type cc) { return mpack[cc]; } + + void set_pack (jit_convention::type cc, convert_fn fn) { mpack[cc] = fn; } + + // The inverse operation of pack. + convert_fn unpack (jit_convention::type cc) { return munpack[cc]; } + + void set_unpack (jit_convention::type cc, convert_fn fn) + { munpack[cc] = fn; } + + // The resulting type after pack is called. + llvm::Type *packed_type (jit_convention::type cc) + { return mpacked_type[cc]; } + + void set_packed_type (jit_convention::type cc, llvm::Type *ty) + { mpacked_type[cc] = ty; } +private: + std::string mname; + jit_type *mparent; + llvm::Type *llvm_type; + int mid; + size_t mdepth; + bool mskip_paren; + + bool msret[jit_convention::length]; + bool mpointer_arg[jit_convention::length]; + + convert_fn mpack[jit_convention::length]; + convert_fn munpack[jit_convention::length]; + + llvm::Type *mpacked_type[jit_convention::length]; +}; + +// seperate print function to allow easy printing if type is null +std::ostream& jit_print (std::ostream& os, jit_type *atype); + +class jit_value; + +// An abstraction for calling llvm functions with jit_values. Deals with +// calling convention details. +class +jit_function +{ + friend std::ostream& operator << (std::ostream& os, const jit_function& fn); +public: + // create a function in an invalid state + jit_function (); + + jit_function (llvm::Module *amodule, jit_convention::type acall_conv, + const llvm::Twine& aname, jit_type *aresult, + const std::vector<jit_type *>& aargs); + + // Use an existing function, but change the argument types. The new argument + // types must behave the same for the current calling convention. + jit_function (const jit_function& fn, jit_type *aresult, + const std::vector<jit_type *>& aargs); + + jit_function (const jit_function& fn); + + // erase the interal LLVM function (if it exists). Will become invalid. + void erase (void); + + template <typename T> + void add_mapping (llvm::ExecutionEngine *engine, T fn) + { + do_add_mapping (engine, reinterpret_cast<void *> (fn)); + } + + bool valid (void) const { return llvm_function; } + + std::string name (void) const; + + llvm::BasicBlock *new_block (const std::string& aname = "body", + llvm::BasicBlock *insert_before = 0); + + llvm::Value *call (llvm::IRBuilderD& builder, + const std::vector<jit_value *>& in_args) const; + + llvm::Value *call (llvm::IRBuilderD& builder, + const std::vector<llvm::Value *>& in_args + = std::vector<llvm::Value *> ()) const; + +#define JIT_PARAM_ARGS llvm::IRBuilderD& builder, +#define JIT_PARAMS builder, +#define JIT_CALL(N) JIT_EXPAND (llvm::Value *, call, llvm::Value *, const, N) + + JIT_CALL (1) + JIT_CALL (2) + JIT_CALL (3) + JIT_CALL (4) + JIT_CALL (5) + +#undef JIT_CALL + +#define JIT_CALL(N) JIT_EXPAND (llvm::Value *, call, jit_value *, const, N) + + JIT_CALL (1); + JIT_CALL (2); + JIT_CALL (3); + +#undef JIT_CALL +#undef JIT_PARAMS +#undef JIT_PARAM_ARGS + + llvm::Value *argument (llvm::IRBuilderD& builder, size_t idx) const; + + void do_return (llvm::IRBuilderD& builder, llvm::Value *rval = 0, + bool verify = true); + + llvm::Function *to_llvm (void) const { return llvm_function; } + + // If true, then the return value is passed as a pointer in the first argument + bool sret (void) const { return mresult && mresult->sret (call_conv); } + + bool can_error (void) const { return mcan_error; } + + void mark_can_error (void) { mcan_error = true; } + + jit_type *result (void) const { return mresult; } + + jit_type *argument_type (size_t idx) const + { + assert (idx < args.size ()); + return args[idx]; + } + + const std::vector<jit_type *>& arguments (void) const { return args; } +private: + void do_add_mapping (llvm::ExecutionEngine *engine, void *fn); + + llvm::Module *module; + llvm::Function *llvm_function; + jit_type *mresult; + std::vector<jit_type *> args; + jit_convention::type call_conv; + bool mcan_error; +}; + +std::ostream& operator << (std::ostream& os, const jit_function& fn); + +// Keeps track of information about how to implement operations (+, -, *, ect) +// and their resulting types. +class +jit_operation +{ +public: + // type signature vector + typedef std::vector<jit_type *> signature_vec; + + virtual ~jit_operation (void); + + void add_overload (const jit_function& func) + { + add_overload (func, func.arguments ()); + } + + void add_overload (const jit_function& func, + const signature_vec& args); + + const jit_function& overload (const signature_vec& types) const; + + jit_type *result (const signature_vec& types) const + { + const jit_function& temp = overload (types); + return temp.result (); + } + +#define JIT_PARAMS +#define JIT_PARAM_ARGS +#define JIT_OVERLOAD(N) \ + JIT_EXPAND (const jit_function&, overload, jit_type *, const, N) \ + JIT_EXPAND (jit_type *, result, jit_type *, const, N) + + JIT_OVERLOAD (1); + JIT_OVERLOAD (2); + JIT_OVERLOAD (3); + +#undef JIT_PARAMS +#undef JIT_PARAM_ARGS + + const std::string& name (void) const { return mname; } + + void stash_name (const std::string& aname) { mname = aname; } +protected: + virtual jit_function *generate (const signature_vec& types) const; +private: + Array<octave_idx_type> to_idx (const signature_vec& types) const; + + const jit_function& do_generate (const signature_vec& types) const; + + struct signature_cmp + { + bool operator () (const signature_vec *lhs, const signature_vec *rhs) const; + }; + + typedef std::map<const signature_vec *, jit_function *, signature_cmp> + generated_map; + + mutable generated_map generated; + + std::vector<Array<jit_function> > overloads; + + std::string mname; +}; + +class +jit_index_operation : public jit_operation +{ +public: + jit_index_operation (void) : module (0), engine (0) { } + + void initialize (llvm::Module *amodule, llvm::ExecutionEngine *aengine) + { + module = amodule; + engine = aengine; + do_initialize (); + } +protected: + virtual jit_function *generate (const signature_vec& types) const; + + virtual jit_function *generate_matrix (const signature_vec& types) const = 0; + + virtual void do_initialize (void) = 0; + + // helper functions + // [start_idx, end_idx). + llvm::Value *create_arg_array (llvm::IRBuilderD& builder, + const jit_function &fn, size_t start_idx, + size_t end_idx) const; + + llvm::Module *module; + llvm::ExecutionEngine *engine; +}; + +class +jit_paren_subsref : public jit_index_operation +{ +protected: + virtual jit_function *generate_matrix (const signature_vec& types) const; + + virtual void do_initialize (void); +private: + jit_function paren_scalar; +}; + +class +jit_paren_subsasgn : public jit_index_operation +{ +protected: + jit_function *generate_matrix (const signature_vec& types) const; + + virtual void do_initialize (void); +private: + jit_function paren_scalar; +}; + +// A singleton class which handles the construction of jit_types and +// jit_operations. +class +jit_typeinfo +{ +public: + static void initialize (llvm::Module *m, llvm::ExecutionEngine *e); + + static jit_type *join (jit_type *lhs, jit_type *rhs) + { + return instance->do_join (lhs, rhs); + } + + static jit_type *get_any (void) { return instance->any; } + + static jit_type *get_matrix (void) { return instance->matrix; } + + static jit_type *get_scalar (void) { return instance->scalar; } + + static llvm::Type *get_scalar_llvm (void) + { return instance->scalar->to_llvm (); } + + static jit_type *get_scalar_ptr (void) { return instance->scalar_ptr; } + + static jit_type *get_any_ptr (void) { return instance->any_ptr; } + + static jit_type *get_range (void) { return instance->range; } + + static jit_type *get_string (void) { return instance->string; } + + static jit_type *get_bool (void) { return instance->boolean; } + + static jit_type *get_index (void) { return instance->index; } + + static llvm::Type *get_index_llvm (void) + { return instance->index->to_llvm (); } + + static jit_type *get_complex (void) { return instance->complex; } + + // Get the jit_type of an octave_value + static jit_type *type_of (const octave_value& ov) + { + return instance->do_type_of (ov); + } + + static const jit_operation& binary_op (int op) + { + return instance->do_binary_op (op); + } + + static const jit_operation& unary_op (int op) + { + return instance->do_unary_op (op); + } + + static const jit_operation& grab (void) { return instance->grab_fn; } + + static const jit_function& get_grab (jit_type *type) + { + return instance->grab_fn.overload (type); + } + + static const jit_operation& release (void) + { + return instance->release_fn; + } + + static const jit_function& get_release (jit_type *type) + { + return instance->release_fn.overload (type); + } + + static const jit_operation& destroy (void) + { + return instance->destroy_fn; + } + + static const jit_operation& print_value (void) + { + return instance->print_fn; + } + + static const jit_operation& for_init (void) + { + return instance->for_init_fn; + } + + static const jit_operation& for_check (void) + { + return instance->for_check_fn; + } + + static const jit_operation& for_index (void) + { + return instance->for_index_fn; + } + + static const jit_operation& make_range (void) + { + return instance->make_range_fn; + } + + static const jit_operation& paren_subsref (void) + { + return instance->paren_subsref_fn; + } + + static const jit_operation& paren_subsasgn (void) + { + return instance->paren_subsasgn_fn; + } + + static const jit_operation& logically_true (void) + { + return instance->logically_true_fn; + } + + static const jit_operation& cast (jit_type *result) + { + return instance->do_cast (result); + } + + static const jit_function& cast (jit_type *to, jit_type *from) + { + return instance->do_cast (to, from); + } + + static llvm::Value *insert_error_check (llvm::IRBuilderD& bld) + { + return instance->do_insert_error_check (bld); + } + + static llvm::Value *insert_interrupt_check (llvm::IRBuilderD& bld) + { + return instance->do_insert_interrupt_check (bld); + } + + static const jit_operation& end (void) + { + return instance->end_fn; + } + + static const jit_function& end (jit_value *value, jit_value *index, + jit_value *count) + { + return instance->do_end (value, index, count); + } + + static const jit_operation& create_undef (void) + { + return instance->create_undef_fn; + } + + static llvm::Value *create_complex (llvm::Value *real, llvm::Value *imag) + { + return instance->complex_new (real, imag); + } +private: + jit_typeinfo (llvm::Module *m, llvm::ExecutionEngine *e); + + // FIXME: Do these methods really need to be in jit_typeinfo? + jit_type *do_join (jit_type *lhs, jit_type *rhs) + { + // empty case + if (! lhs) + return rhs; + + if (! rhs) + return lhs; + + // check for a shared parent + while (lhs != rhs) + { + if (lhs->depth () > rhs->depth ()) + lhs = lhs->parent (); + else if (lhs->depth () < rhs->depth ()) + rhs = rhs->parent (); + else + { + // we MUST have depth > 0 as any is the base type of everything + do + { + lhs = lhs->parent (); + rhs = rhs->parent (); + } + while (lhs != rhs); + } + } + + return lhs; + } + + jit_type *do_difference (jit_type *lhs, jit_type *) + { + // FIXME: Maybe we can do something smarter? + return lhs; + } + + jit_type *do_type_of (const octave_value &ov) const; + + const jit_operation& do_binary_op (int op) const + { + assert (static_cast<size_t>(op) < binary_ops.size ()); + return binary_ops[op]; + } + + const jit_operation& do_unary_op (int op) const + { + assert (static_cast<size_t> (op) < unary_ops.size ()); + return unary_ops[op]; + } + + const jit_operation& do_cast (jit_type *to) + { + static jit_operation null_function; + if (! to) + return null_function; + + size_t id = to->type_id (); + if (id >= casts.size ()) + return null_function; + return casts[id]; + } + + const jit_function& do_cast (jit_type *to, jit_type *from) + { + return do_cast (to).overload (from); + } + + const jit_function& do_end (jit_value *value, jit_value *index, + jit_value *count); + + jit_type *new_type (const std::string& name, jit_type *parent, + llvm::Type *llvm_type, bool skip_paren = false); + + void add_print (jit_type *ty, void *fptr); + + void add_binary_op (jit_type *ty, int op, int llvm_op); + + void add_binary_icmp (jit_type *ty, int op, int llvm_op); + + void add_binary_fcmp (jit_type *ty, int op, int llvm_op); + + // create a function with an external calling convention + // forces the function pointer to be specified + template <typename T> + jit_function create_external (llvm::ExecutionEngine *ee, T fn, + const llvm::Twine& name, jit_type *ret, + const std::vector<jit_type *>& args + = std::vector<jit_type *> ()) + { + jit_function retval = create_function (jit_convention::external, name, ret, + args); + retval.add_mapping (ee, fn); + return retval; + } + +#define JIT_PARAM_ARGS llvm::ExecutionEngine *ee, T fn, \ + const llvm::Twine& name, jit_type *ret, +#define JIT_PARAMS ee, fn, name, ret, +#define CREATE_FUNCTION(N) JIT_EXPAND(template <typename T> jit_function, \ + create_external, \ + jit_type *, /* empty */, N) + + CREATE_FUNCTION(1); + CREATE_FUNCTION(2); + CREATE_FUNCTION(3); + CREATE_FUNCTION(4); + +#undef JIT_PARAM_ARGS +#undef JIT_PARAMS +#undef CREATE_FUNCTION + + // use create_external or create_internal directly + jit_function create_function (jit_convention::type cc, + const llvm::Twine& name, jit_type *ret, + const std::vector<jit_type *>& args + = std::vector<jit_type *> ()); + + // create an internal calling convention (a function defined in llvm) + jit_function create_internal (const llvm::Twine& name, jit_type *ret, + const std::vector<jit_type *>& args + = std::vector<jit_type *> ()) + { + return create_function (jit_convention::internal, name, ret, args); + } + +#define JIT_PARAM_ARGS const llvm::Twine& name, jit_type *ret, +#define JIT_PARAMS name, ret, +#define CREATE_FUNCTION(N) JIT_EXPAND(jit_function, create_internal, \ + jit_type *, /* empty */, N) + + CREATE_FUNCTION(1); + CREATE_FUNCTION(2); + CREATE_FUNCTION(3); + CREATE_FUNCTION(4); + +#undef JIT_PARAM_ARGS +#undef JIT_PARAMS +#undef CREATE_FUNCTION + + jit_function create_identity (jit_type *type); + + llvm::Value *do_insert_error_check (llvm::IRBuilderD& bld); + + llvm::Value *do_insert_interrupt_check (llvm::IRBuilderD& bld); + + void add_builtin (const std::string& name); + + void register_intrinsic (const std::string& name, size_t id, + jit_type *result, jit_type *arg0) + { + std::vector<jit_type *> args (1, arg0); + register_intrinsic (name, id, result, args); + } + + void register_intrinsic (const std::string& name, size_t id, jit_type *result, + const std::vector<jit_type *>& args); + + void register_generic (const std::string& name, jit_type *result, + jit_type *arg0) + { + std::vector<jit_type *> args (1, arg0); + register_generic (name, result, args); + } + + void register_generic (const std::string& name, jit_type *result, + const std::vector<jit_type *>& args); + + octave_builtin *find_builtin (const std::string& name); + + jit_function mirror_binary (const jit_function& fn); + + llvm::Function *wrap_complex (llvm::Function *wrap); + + static llvm::Value *pack_complex (llvm::IRBuilderD& bld, + llvm::Value *cplx); + + static llvm::Value *unpack_complex (llvm::IRBuilderD& bld, + llvm::Value *result); + + llvm::Value *complex_real (llvm::Value *cx); + + llvm::Value *complex_real (llvm::Value *cx, llvm::Value *real); + + llvm::Value *complex_imag (llvm::Value *cx); + + llvm::Value *complex_imag (llvm::Value *cx, llvm::Value *imag); + + llvm::Value *complex_new (llvm::Value *real, llvm::Value *imag); + + void create_int (size_t nbits); + + jit_type *intN (size_t nbits) const; + + static jit_typeinfo *instance; + + llvm::Module *module; + llvm::ExecutionEngine *engine; + int next_id; + + llvm::GlobalVariable *lerror_state; + llvm::GlobalVariable *loctave_interrupt_state; + + llvm::Type *sig_atomic_type; + + std::vector<jit_type*> id_to_type; + jit_type *any; + jit_type *matrix; + jit_type *scalar; + jit_type *scalar_ptr; // a fake type for interfacing with C++ + jit_type *any_ptr; // a fake type for interfacing with C++ + jit_type *range; + jit_type *string; + jit_type *boolean; + jit_type *index; + jit_type *complex; + jit_type *unknown_function; + std::map<size_t, jit_type *> ints; + std::map<std::string, jit_type *> builtins; + + llvm::StructType *complex_ret; + + std::vector<jit_operation> binary_ops; + std::vector<jit_operation> unary_ops; + jit_operation grab_fn; + jit_operation release_fn; + jit_operation destroy_fn; + jit_operation print_fn; + jit_operation for_init_fn; + jit_operation for_check_fn; + jit_operation for_index_fn; + jit_operation logically_true_fn; + jit_operation make_range_fn; + jit_paren_subsref paren_subsref_fn; + jit_paren_subsasgn paren_subsasgn_fn; + jit_operation end1_fn; + jit_operation end_fn; + jit_operation create_undef_fn; + + jit_function any_call; + + // type id -> cast function TO that type + std::vector<jit_operation> casts; + + // type id -> identity function + std::vector<jit_function> identities; + + llvm::IRBuilderD& builder; +}; + +#endif +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-util.cc Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,51 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +// defines required by llvm +#define __STDC_LIMIT_MACROS +#define __STDC_CONSTANT_MACROS + +#if defined (HAVE_CONFIG_H) +# include "config.h" +#endif + +#if defined (HAVE_LLVM) + +#if defined (HAVE_LLVM_IR_FUNCTION_H) +# include <llvm/IR/Value.h> +#else +# include <llvm/Value.h> +#endif + +#include <llvm/Support/raw_os_ostream.h> + +std::ostream& +operator<< (std::ostream& os, const llvm::Value& v) +{ + llvm::raw_os_ostream llvm_out (os); + v.print (llvm_out); + return os; +} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/jit-util.h Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,219 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +// Some utility classes and functions used throughout jit + +#if ! defined (octave_jit_util_h) +#define octave_jit_util_h 1 + +#include "octave-config.h" + +#if defined (HAVE_LLVM) + +#include <stdexcept> + +#if defined (HAVE_LLVM_IR_DATALAYOUT_H) || defined (HAVE_LLVM_DATALAYOUT_H) +# define HAVE_LLVM_DATALAYOUT +#endif + +// we don't want to include llvm headers here, as they require +// __STDC_LIMIT_MACROS and __STDC_CONSTANT_MACROS be defined in the entire +// compilation unit +namespace llvm +{ + class Value; + class Module; +#if defined (LEGACY_PASSMANAGER) + namespace legacy + { + class FunctionPassManager; + class PassManager; + } +#else + class FunctionPassManager; + class PassManager; +#endif + class ExecutionEngine; + class Function; + class BasicBlock; + class LLVMContext; + class Type; + class StructType; + class Twine; + class GlobalVariable; + class TerminatorInst; + class PHINode; + + class ConstantFolder; + + template <bool preserveNames> + class IRBuilderDefaultInserter; + + template <bool preserveNames, typename T, typename Inserter> + class IRBuilder; + +typedef IRBuilder<true, ConstantFolder, IRBuilderDefaultInserter<true> > +IRBuilderD; +} + +class octave_base_value; +class octave_builtin; +class octave_value; +class tree; +class tree_expression; + +// thrown when we should give up on JIT and interpret +class jit_fail_exception : public std::runtime_error +{ +public: + jit_fail_exception (void) : std::runtime_error ("unknown"), mknown (false) { } + jit_fail_exception (const std::string& reason) : std::runtime_error (reason), + mknown (true) + { } + + bool known (void) const { return mknown; } +private: + bool mknown; +}; + +// llvm doesn't provide this, and it's really useful for debugging +std::ostream& operator<< (std::ostream& os, const llvm::Value& v); + +template <typename HOLDER_T, typename SUB_T> +class jit_internal_node; + +// jit_internal_list and jit_internal_node implement generic embedded doubly +// linked lists. List items extend from jit_internal_list, and can be placed +// in nodes of type jit_internal_node. We use CRTP twice. +template <typename LIST_T, typename NODE_T> +class +jit_internal_list +{ + friend class jit_internal_node<LIST_T, NODE_T>; +public: + jit_internal_list (void) : use_head (0), use_tail (0), muse_count (0) { } + + virtual ~jit_internal_list (void) + { + while (use_head) + use_head->stash_value (0); + } + + NODE_T *first_use (void) const { return use_head; } + + size_t use_count (void) const { return muse_count; } +private: + NODE_T *use_head; + NODE_T *use_tail; + size_t muse_count; +}; + +// a node for internal linked lists +template <typename LIST_T, typename NODE_T> +class +jit_internal_node +{ +public: + typedef jit_internal_list<LIST_T, NODE_T> jit_ilist; + + jit_internal_node (void) : mvalue (0), mnext (0), mprev (0) { } + + ~jit_internal_node (void) { remove (); } + + LIST_T *value (void) const { return mvalue; } + + void stash_value (LIST_T *avalue) + { + remove (); + + mvalue = avalue; + + if (mvalue) + { + jit_ilist *ilist = mvalue; + NODE_T *sthis = static_cast<NODE_T *> (this); + if (ilist->use_head) + { + ilist->use_tail->mnext = sthis; + mprev = ilist->use_tail; + } + else + ilist->use_head = sthis; + + ilist->use_tail = sthis; + ++ilist->muse_count; + } + } + + NODE_T *next (void) const { return mnext; } + + NODE_T *prev (void) const { return mprev; } +private: + void remove () + { + if (mvalue) + { + jit_ilist *ilist = mvalue; + if (mprev) + mprev->mnext = mnext; + else + // we are the use_head + ilist->use_head = mnext; + + if (mnext) + mnext->mprev = mprev; + else + // we are the use tail + ilist->use_tail = mprev; + + mnext = mprev = 0; + --ilist->muse_count; + mvalue = 0; + } + } + + LIST_T *mvalue; + NODE_T *mnext; + NODE_T *mprev; +}; + +// Use like: isa<jit_phi> (value) +// basically just a short cut type typing dyanmic_cast. +template <typename T, typename U> +bool isa (U *value) +{ + return dynamic_cast<T *> (value); +} + +#define JIT_ASSIGN_ARG(i) the_args[i] = arg ## i; +#define JIT_EXPAND(ret, fname, type, isconst, N) \ + ret fname (JIT_PARAM_ARGS OCT_MAKE_DECL_LIST (type, arg, N)) isconst \ + { \ + std::vector<type> the_args (N); \ + OCT_ITERATE_MACRO (JIT_ASSIGN_ARG, N); \ + return fname (JIT_PARAMS the_args); \ + } + +#endif +#endif
--- a/libinterp/parse-tree/module.mk Tue Jan 31 17:42:29 2017 -0500 +++ b/libinterp/parse-tree/module.mk Tue Jan 31 17:43:58 2017 -0500 @@ -1,4 +1,7 @@ PARSE_TREE_INC = \ + libinterp/parse-tree/jit-ir.h \ + libinterp/parse-tree/jit-typeinfo.h \ + libinterp/parse-tree/jit-util.h \ libinterp/parse-tree/lex.h \ libinterp/parse-tree/parse.h \ libinterp/parse-tree/pt-all.h \ @@ -22,6 +25,7 @@ libinterp/parse-tree/pt-funcall.h \ libinterp/parse-tree/pt-id.h \ libinterp/parse-tree/pt-idx.h \ + libinterp/parse-tree/pt-jit.h \ libinterp/parse-tree/pt-jump.h \ libinterp/parse-tree/pt-loop.h \ libinterp/parse-tree/pt-mat.h \ @@ -34,10 +38,14 @@ libinterp/parse-tree/pt.h \ libinterp/parse-tree/token.h + ## oct-gperf.h and oct-parse.h are in the SRC list so that they will ## be distributed but not installed. PARSE_TREE_SRC = \ + libinterp/parse-tree/jit-ir.cc \ + libinterp/parse-tree/jit-typeinfo.cc \ + libinterp/parse-tree/jit-util.cc \ libinterp/parse-tree/lex.ll \ libinterp/parse-tree/oct-gperf.h \ libinterp/parse-tree/oct-parse.h \ @@ -62,6 +70,7 @@ libinterp/parse-tree/pt-funcall.cc \ libinterp/parse-tree/pt-id.cc \ libinterp/parse-tree/pt-idx.cc \ + libinterp/parse-tree/pt-jit.cc \ libinterp/parse-tree/pt-jump.cc \ libinterp/parse-tree/pt-loop.cc \ libinterp/parse-tree/pt-mat.cc \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/pt-jit.cc Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,2607 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +#define __STDC_LIMIT_MACROS +#define __STDC_CONSTANT_MACROS + +#if defined (HAVE_CONFIG_H) +# include "config.h" +#endif + +#include "debug.h" +#include "defun.h" +#include "errwarn.h" +#include "ov.h" +#include "pt-all.h" +#include "pt-jit.h" +#include "sighandlers.h" +#include "symtab.h" +#include "variables.h" + +#if defined (HAVE_LLVM) + +static bool Vdebug_jit = false; + +static bool Vjit_enable = false; + +static int Vjit_startcnt = 1000; + +static int Vjit_failcnt = 0; + +#include <llvm/Analysis/CallGraph.h> +#include <llvm/Analysis/Passes.h> + +#if defined (HAVE_LLVM_IR_VERIFIER_H) +# include <llvm/IR/Verifier.h> +#else +# include <llvm/Analysis/Verifier.h> +#endif + +#include <llvm/Bitcode/ReaderWriter.h> +#include <llvm/ExecutionEngine/ExecutionEngine.h> +#include <llvm/ExecutionEngine/JIT.h> + +#if defined (LEGACY_PASSMANAGER) +# include <llvm/IR/LegacyPassManager.h> +#else +# include <llvm/PassManager.h> +#endif + +#if defined (HAVE_LLVM_IR_FUNCTION_H) +# include <llvm/IR/LLVMContext.h> +# include <llvm/IR/Module.h> +#else +# include <llvm/LLVMContext.h> +# include <llvm/Module.h> +#endif + +#if defined (HAVE_LLVM_SUPPORT_IRBUILDER_H) +# include <llvm/Support/IRBuilder.h> +#elif defined(HAVE_LLVM_IR_IRBUILDER_H) +# include <llvm/IR/IRBuilder.h> +#else +# include <llvm/IRBuilder.h> +#endif + +#include <llvm/Support/raw_os_ostream.h> +#include <llvm/Support/TargetSelect.h> + +#if defined (HAVE_LLVM_IR_DATALAYOUT_H) +# include <llvm/IR/DataLayout.h> +#elif defined(HAVE_LLVM_DATALAYOUT_H) +# include <llvm/DataLayout.h> +#else +# include <llvm/Target/TargetData.h> +#endif + +#include <llvm/Transforms/IPO.h> +#include <llvm/Transforms/Scalar.h> + +static llvm::IRBuilder<> builder (llvm::getGlobalContext ()); + +static llvm::LLVMContext& context = llvm::getGlobalContext (); + +// -------------------- jit_break_exception -------------------- + +// jit_break is thrown whenever a branch we are converting has only breaks or +// continues. This is because all code that follows a break or continue +// is dead. +class jit_break_exception : public std::exception +{ }; + +// -------------------- jit_convert -------------------- +jit_convert::jit_convert (tree &tee, jit_type *for_bounds) + : converting_function (false) +{ + initialize (symbol_table::current_scope ()); + + if (for_bounds) + create_variable (next_for_bounds (false), for_bounds); + + try + { + visit (tee); + } + catch (const jit_break_exception&) + { } + + // breaks must have been handled by the top level loop + assert (breaks.empty ()); + assert (continues.empty ()); + + block->append (factory.create<jit_branch> (final_block)); + blocks.push_back (final_block); + + for (variable_map::iterator iter = vmap.begin (); iter != vmap.end (); ++iter) + { + jit_variable *var = iter->second; + const std::string& name = var->name (); + if (name.size () && name[0] != '#') + final_block->append (factory.create<jit_store_argument> (var)); + } + + final_block->append (factory.create<jit_return> ()); +} + +jit_convert::jit_convert (octave_user_function& fcn, + const std::vector<jit_type *>& args) + : converting_function (true) +{ + initialize (fcn.scope ()); + + tree_parameter_list *plist = fcn.parameter_list (); + tree_parameter_list *rlist = fcn.return_list (); + if (plist && plist->takes_varargs ()) + throw jit_fail_exception ("varags not supported"); + + if (rlist && (rlist->size () > 1 || rlist->takes_varargs ())) + throw jit_fail_exception ("multiple returns not supported"); + + if (plist) + { + tree_parameter_list::iterator piter = plist->begin (); + for (size_t i = 0; i < args.size (); ++i, ++piter) + { + if (piter == plist->end ()) + throw jit_fail_exception ("Too many parameter to function"); + + tree_decl_elt *elt = *piter; + std::string name = elt->name (); + create_variable (name, args[i]); + } + } + + jit_value *return_value = 0; + bool all_breaking = false; + if (fcn.is_special_expr ()) + { + tree_expression *expr = fcn.special_expr (); + if (expr) + { + jit_variable *retvar = get_variable ("#return"); + jit_value *retval = 0; + try + { + retval = visit (expr); + } + catch (const jit_break_exception&) + { } + + if (breaks.size () || continues.size ()) + throw jit_fail_exception ("break/continue not supported in " + "anonymous functions"); + + block->append (factory.create<jit_assign> (retvar, retval)); + return_value = retvar; + } + } + else + { + try + { + visit_statement_list (*fcn.body ()); + } + catch (const jit_break_exception&) + { + all_breaking = true; + } + + // the user may use break or continue to exit the function + finish_breaks (final_block, continues); + finish_breaks (final_block, breaks); + } + + if (! all_breaking) + block->append (factory.create<jit_branch> (final_block)); + + blocks.push_back (final_block); + block = final_block; + + if (! return_value && rlist && rlist->size () == 1) + { + tree_decl_elt *elt = rlist->front (); + return_value = get_variable (elt->name ()); + } + + // FIXME: We should use live range analysis to delete variables where needed. + // For now we just delete everything at the end of the function. + for (variable_map::iterator iter = vmap.begin (); iter != vmap.end (); ++iter) + { + if (iter->second != return_value) + { + jit_call *call; + call = factory.create<jit_call> (&jit_typeinfo::destroy, + iter->second); + final_block->append (call); + } + } + + if (return_value) + final_block->append (factory.create<jit_return> (return_value)); + else + final_block->append (factory.create<jit_return> ()); +} + +void +jit_convert::visit_anon_fcn_handle (tree_anon_fcn_handle&) +{ + throw jit_fail_exception ("No visit_anon_fcn_handle implementation"); +} + +void +jit_convert::visit_argument_list (tree_argument_list&) +{ + throw jit_fail_exception ("No visit_argument_list implementation"); +} + +void +jit_convert::visit_binary_expression (tree_binary_expression& be) +{ + if (be.op_type () >= octave_value::num_binary_ops) + { + tree_boolean_expression *boole; + boole = dynamic_cast<tree_boolean_expression *> (&be); + assert (boole); + bool is_and = boole->op_type () == tree_boolean_expression::bool_and; + + std::string short_name = next_shortcircut_result (); + jit_variable *short_result = factory.create<jit_variable> (short_name); + vmap[short_name] = short_result; + + jit_block *done = factory.create<jit_block> (block->name ()); + tree_expression *lhs = be.lhs (); + jit_value *lhsv = visit (lhs); + lhsv = create_checked (&jit_typeinfo::logically_true, lhsv); + + jit_block *short_early = factory.create<jit_block> ("short_early"); + blocks.push_back (short_early); + + jit_block *short_cont = factory.create<jit_block> ("short_cont"); + + if (is_and) + block->append (factory.create<jit_cond_branch> (lhsv, short_cont, + short_early)); + else + block->append (factory.create<jit_cond_branch> (lhsv, short_early, + short_cont)); + + block = short_early; + + jit_value *early_result = factory.create<jit_const_bool> (! is_and); + block->append (factory.create<jit_assign> (short_result, early_result)); + block->append (factory.create<jit_branch> (done)); + + blocks.push_back (short_cont); + block = short_cont; + + tree_expression *rhs = be.rhs (); + jit_value *rhsv = visit (rhs); + rhsv = create_checked (&jit_typeinfo::logically_true, rhsv); + block->append (factory.create<jit_assign> (short_result, rhsv)); + block->append (factory.create<jit_branch> (done)); + + blocks.push_back (done); + block = done; + result = short_result; + } + else + { + tree_expression *lhs = be.lhs (); + jit_value *lhsv = visit (lhs); + + tree_expression *rhs = be.rhs (); + jit_value *rhsv = visit (rhs); + + const jit_operation& fn = jit_typeinfo::binary_op (be.op_type ()); + result = create_checked (fn, lhsv, rhsv); + } +} + +void +jit_convert::visit_break_command (tree_break_command&) +{ + breaks.push_back (block); + throw jit_break_exception (); +} + +void +jit_convert::visit_colon_expression (tree_colon_expression& expr) +{ + // in the futher we need to add support for classes and deal with rvalues + jit_value *base = visit (expr.base ()); + jit_value *limit = visit (expr.limit ()); + jit_value *increment; + tree_expression *tinc = expr.increment (); + + if (tinc) + increment = visit (tinc); + else + increment = factory.create<jit_const_scalar> (1); + + result = block->append (factory.create<jit_call> (jit_typeinfo::make_range, + base, limit, increment)); +} + +void +jit_convert::visit_continue_command (tree_continue_command&) +{ + continues.push_back (block); + throw jit_break_exception (); +} + +void +jit_convert::visit_global_command (tree_global_command&) +{ + throw jit_fail_exception ("No visit_global_command implemenation"); +} + +void +jit_convert::visit_persistent_command (tree_persistent_command&) +{ + throw jit_fail_exception ("No visit_persistent_command implementation"); +} + +void +jit_convert::visit_decl_elt (tree_decl_elt&) +{ + throw jit_fail_exception ("No visit_decl_elt implementation"); +} + +void +jit_convert::visit_decl_init_list (tree_decl_init_list&) +{ + throw jit_fail_exception ("No visit_decl_init_list implementation"); +} + +void +jit_convert::visit_simple_for_command (tree_simple_for_command& cmd) +{ + // Note we do an initial check to see if the loop will run atleast once. + // This allows us to get better type inference bounds on variables defined + // and used only inside the for loop (e.g., the index variable) + + // If we are a nested for loop we need to store the previous breaks + octave::unwind_protect frame; + frame.protect_var (breaks); + frame.protect_var (continues); + breaks.clear (); + continues.clear (); + + // we need a variable for our iterator, because it is used in multiple blocks + std::string iter_name = next_iterator (); + jit_variable *iterator = factory.create<jit_variable> (iter_name); + factory.create<jit_variable> (iter_name); + vmap[iter_name] = iterator; + + jit_block *body = factory.create<jit_block> ("for_body"); + jit_block *tail = factory.create<jit_block> ("for_tail"); + + // do control expression, iter init, and condition check in prev_block (block) + // if we are the top level for loop, the bounds is an input argument. + jit_value *control = find_variable (next_for_bounds ()); + if (! control) + control = visit (cmd.control_expr ()); + jit_call *init_iter = factory.create<jit_call> (jit_typeinfo::for_init, + control); + block->append (init_iter); + block->append (factory.create<jit_assign> (iterator, init_iter)); + + jit_call *check = factory.create<jit_call> (jit_typeinfo::for_check, control, + iterator); + block->append (check); + block->append (factory.create<jit_cond_branch> (check, body, tail)); + + blocks.push_back (body); + block = body; + + // compute the syntactical iterator + jit_call *idx_rhs = factory.create<jit_call> (jit_typeinfo::for_index, + control, iterator); + block->append (idx_rhs); + do_assign (cmd.left_hand_side (), idx_rhs); + + // do loop + tree_statement_list *pt_body = cmd.body (); + bool all_breaking = false; + try + { + pt_body->accept (*this); + } + catch (const jit_break_exception&) + { + if (continues.empty ()) + { + // WTF are you doing user? Every branch was a break, why did you have + // a loop??? Users are silly people... + finish_breaks (tail, breaks); + blocks.push_back (tail); + block = tail; + return; + } + + all_breaking = true; + } + + // check our condition, continues jump to this block + jit_block *check_block = factory.create<jit_block> ("for_check"); + blocks.push_back (check_block); + + jit_block *interrupt_check = factory.create<jit_block> ("for_interrupt"); + blocks.push_back (interrupt_check); + + if (! all_breaking) + block->append (factory.create<jit_branch> (check_block)); + finish_breaks (check_block, continues); + + block = check_block; + const jit_operation& add_fn = jit_typeinfo::binary_op (octave_value::op_add); + jit_value *one = factory.create<jit_const_index> (1); + jit_call *iter_inc = factory.create<jit_call> (add_fn, iterator, one); + block->append (iter_inc); + block->append (factory.create<jit_assign> (iterator, iter_inc)); + check = block->append (factory.create<jit_call> (jit_typeinfo::for_check, + control, iterator)); + block->append (factory.create<jit_cond_branch> (check, interrupt_check, + tail)); + + block = interrupt_check; + jit_error_check *ec + = factory.create<jit_error_check> (jit_error_check::var_interrupt, + body, final_block); + block->append (ec); + + // breaks will go to our tail + blocks.push_back (tail); + finish_breaks (tail, breaks); + block = tail; +} + +void +jit_convert::visit_complex_for_command (tree_complex_for_command&) +{ + throw jit_fail_exception ("No visit_complex_for_command implementation"); +} + +void +jit_convert::visit_octave_user_script (octave_user_script&) +{ + throw jit_fail_exception ("No visit_octave_user_script implementation"); +} + +void +jit_convert::visit_octave_user_function (octave_user_function&) +{ + throw jit_fail_exception ("No visit_octave_user_function implementation"); +} + +void +jit_convert::visit_octave_user_function_header (octave_user_function&) +{ + throw jit_fail_exception ("No visit_octave_user_function_header implementation"); +} + +void +jit_convert::visit_octave_user_function_trailer (octave_user_function&) +{ + throw jit_fail_exception ("No visit_octave_user_function_trailer implementation"); +} + +void +jit_convert::visit_function_def (tree_function_def&) +{ + throw jit_fail_exception ("No visit_function_def implementation"); +} + +void +jit_convert::visit_identifier (tree_identifier& ti) +{ + if (ti.has_magic_end ()) + { + if (! end_context.size ()) + throw jit_fail_exception ("Illegal end"); + result = block->append (factory.create<jit_magic_end> (end_context)); + } + else + { + jit_variable *var = get_variable (ti.name ()); + jit_instruction *instr; + instr = factory.create<jit_call> (&jit_typeinfo::grab, var); + result = block->append (instr); + } +} + +void +jit_convert::visit_if_clause (tree_if_clause&) +{ + throw jit_fail_exception ("No visit_if_clause implementation"); +} + +void +jit_convert::visit_if_command (tree_if_command& cmd) +{ + tree_if_command_list *lst = cmd.cmd_list (); + assert (lst); // jwe: Can this be null? + lst->accept (*this); +} + +void +jit_convert::visit_if_command_list (tree_if_command_list& lst) +{ + tree_if_clause *last = lst.back (); + size_t last_else = static_cast<size_t> (last->is_else_clause ()); + + // entry_blocks represents the block you need to enter in order to execute + // the condition check for the ith clause. For the else, it is simple the + // else body. If there is no else body, then it is padded with the tail. + std::vector<jit_block *> entry_blocks (lst.size () + 1 - last_else); + entry_blocks[0] = block; + + // we need to construct blocks first, because they have jumps to each other. + tree_if_command_list::iterator iter = lst.begin (); + ++iter; + for (size_t i = 1; iter != lst.end (); ++iter, ++i) + { + tree_if_clause *tic = *iter; + if (tic->is_else_clause ()) + entry_blocks[i] = factory.create<jit_block> ("else"); + else + entry_blocks[i] = factory.create<jit_block> ("ifelse_cond"); + } + + jit_block *tail = factory.create<jit_block> ("if_tail"); + if (! last_else) + entry_blocks[entry_blocks.size () - 1] = tail; + + // each branch in the if statement will have different breaks/continues + block_list current_breaks = breaks; + block_list current_continues = continues; + breaks.clear (); + continues.clear (); + + size_t num_incomming = 0; // number of incomming blocks to our tail + iter = lst.begin (); + for (size_t i = 0; iter != lst.end (); ++iter, ++i) + { + tree_if_clause *tic = *iter; + block = entry_blocks[i]; + assert (block); + + if (i) // the first block is prev_block, so it has already been added + blocks.push_back (entry_blocks[i]); + + if (! tic->is_else_clause ()) + { + tree_expression *expr = tic->condition (); + jit_value *cond = visit (expr); + jit_call *check = create_checked (&jit_typeinfo::logically_true, + cond); + jit_block *body = factory.create<jit_block> (i == 0 ? "if_body" + : "ifelse_body"); + blocks.push_back (body); + + jit_instruction *br = factory.create<jit_cond_branch> (check, body, + entry_blocks[i + 1]); + block->append (br); + block = body; + } + + tree_statement_list *stmt_lst = tic->commands (); + assert (stmt_lst); // jwe: Can this be null? + + try + { + stmt_lst->accept (*this); + ++num_incomming; + block->append (factory.create<jit_branch> (tail)); + } + catch (const jit_break_exception&) + { } + + current_breaks.splice (current_breaks.end (), breaks); + current_continues.splice (current_continues.end (), continues); + } + + breaks.splice (breaks.end (), current_breaks); + continues.splice (continues.end (), current_continues); + + if (num_incomming || ! last_else) + { + blocks.push_back (tail); + block = tail; + } + else + // every branch broke, so we don't have a tail + throw jit_break_exception (); +} + +void +jit_convert::visit_index_expression (tree_index_expression& exp) +{ + result = resolve (exp); +} + +void +jit_convert::visit_matrix (tree_matrix&) +{ + throw jit_fail_exception ("No visit_matrix implementation"); +} + +void +jit_convert::visit_cell (tree_cell&) +{ + throw jit_fail_exception ("No visit_cell implementation"); +} + +void +jit_convert::visit_multi_assignment (tree_multi_assignment&) +{ + throw jit_fail_exception ("No visit_multi_assignment implementation"); +} + +void +jit_convert::visit_no_op_command (tree_no_op_command&) +{ + throw jit_fail_exception ("No visit_no_op_command implementation"); +} + +void +jit_convert::visit_constant (tree_constant& tc) +{ + octave_value v = tc.rvalue1 (); + jit_type *ty = jit_typeinfo::type_of (v); + + if (ty == jit_typeinfo::get_scalar ()) + { + double dv = v.double_value (); + result = factory.create<jit_const_scalar> (dv); + } + else if (ty == jit_typeinfo::get_range ()) + { + Range rv = v.range_value (); + result = factory.create<jit_const_range> (rv); + } + else if (ty == jit_typeinfo::get_complex ()) + { + Complex cv = v.complex_value (); + result = factory.create<jit_const_complex> (cv); + } + else + throw jit_fail_exception ("Unknown constant"); +} + +void +jit_convert::visit_fcn_handle (tree_fcn_handle&) +{ + throw jit_fail_exception ("No visit_fcn_handle implementation"); +} + +void +jit_convert::visit_funcall (tree_funcall&) +{ + throw jit_fail_exception (); +} + +void +jit_convert::visit_parameter_list (tree_parameter_list&) +{ + throw jit_fail_exception ("No visit_parameter_list implementation"); +} + +void +jit_convert::visit_postfix_expression (tree_postfix_expression& tpe) +{ + octave_value::unary_op etype = tpe.op_type (); + tree_expression *operand = tpe.operand (); + jit_value *operandv = visit (operand); + + const jit_operation& fn = jit_typeinfo::unary_op (etype); + result = create_checked (fn, operandv); + + if (etype == octave_value::op_incr || etype == octave_value::op_decr) + { + jit_value *ret = create_checked (&jit_typeinfo::grab, operandv); + do_assign (operand, result); + result = ret; + } +} + +void +jit_convert::visit_prefix_expression (tree_prefix_expression& tpe) +{ + octave_value::unary_op etype = tpe.op_type (); + tree_expression *operand = tpe.operand (); + const jit_operation& fn = jit_typeinfo::unary_op (etype); + result = create_checked (fn, visit (operand)); + + if (etype == octave_value::op_incr || etype == octave_value::op_decr) + do_assign (operand, result); +} + +void +jit_convert::visit_return_command (tree_return_command&) +{ + throw jit_fail_exception ("No visit_return_command implementation"); +} + +void +jit_convert::visit_return_list (tree_return_list&) +{ + throw jit_fail_exception ("No visit_return_list implementation"); +} + +void +jit_convert::visit_simple_assignment (tree_simple_assignment& tsa) +{ + tree_expression *rhs = tsa.right_hand_side (); + jit_value *rhsv = visit (rhs); + octave_value::assign_op op = tsa.op_type (); + + if (op != octave_value::op_asn_eq) + { + // Do the equivalent binary operation, then assign. + // This is always correct, but it isn't always optimal. + tree_expression *lhs = tsa.left_hand_side (); + jit_value *lhsv = visit (lhs); + octave_value::binary_op bop = octave_value::assign_op_to_binary_op (op); + const jit_operation& fn = jit_typeinfo::binary_op (bop); + rhsv = create_checked (fn, lhsv, rhsv); + } + + result = do_assign (tsa.left_hand_side (), rhsv); +} + +void +jit_convert::visit_statement (tree_statement& stmt) +{ + tree_command *cmd = stmt.command (); + tree_expression *expr = stmt.expression (); + + if (cmd) + visit (cmd); + else + { + // stolen from octave::tree_evaluator::visit_statement + bool do_bind_ans = false; + + if (expr->is_identifier ()) + { + tree_identifier *id = dynamic_cast<tree_identifier *> (expr); + + do_bind_ans = (! id->is_variable ()); + } + else + do_bind_ans = (! expr->is_assignment_expression ()); + + jit_value *expr_result = visit (expr); + + if (do_bind_ans) + do_assign ("ans", expr_result, expr->print_result ()); + else if (expr->is_identifier () && expr->print_result ()) + { + // FIXME: ugly hack, we need to come up with a way to pass + // nargout to visit_identifier + const jit_operation& fn = jit_typeinfo::print_value (); + jit_const_string *name = factory.create<jit_const_string> + (expr->name ()); + block->append (factory.create<jit_call> (fn, name, expr_result)); + } + } +} + +void +jit_convert::visit_statement_list (tree_statement_list& lst) +{ + for (tree_statement_list::iterator iter = lst.begin (); iter != lst.end(); + ++iter) + { + tree_statement *elt = *iter; + // jwe: Can this ever be null? + assert (elt); + elt->accept (*this); + } +} + +void +jit_convert::visit_switch_case (tree_switch_case&) +{ + throw jit_fail_exception ("No visit_switch_case implementation"); +} + +void +jit_convert::visit_switch_case_list (tree_switch_case_list&) +{ + throw jit_fail_exception ("No visit_switch_case_list implementation"); +} + +void +jit_convert::visit_switch_command (tree_switch_command& cmd) +{ + tree_switch_case_list *lst = cmd.case_list (); + + // always visit switch expression + tree_expression *expr = cmd.switch_value (); + assert (expr && "Switch value can not be null"); + jit_value *value = visit (expr); + assert (value); + + size_t case_blocks_num = lst->size (); + + if (! case_blocks_num) // there's nothing to do + return; + + // check for otherwise, it's interpreted as last 'else' condition + size_t has_otherwise = 0; + tree_switch_case *last = lst->back (); + if (last->is_default_case ()) + has_otherwise = 1; + + std::vector<jit_block *> entry_blocks (case_blocks_num + 1 - has_otherwise); + + // the first entry point is always the actual block. Afterward, new blocks + // are created for every case and the otherwise branch + entry_blocks[0] = block; + for (size_t i = 1; i < case_blocks_num; ++i) + entry_blocks[i] = factory.create<jit_block> ("case_cond"); + + jit_block *tail = factory.create<jit_block> ("switch_tail"); + + // if there's no otherwise branch, the 'else' of the last branch + // has to point to the tail + if (! has_otherwise) + entry_blocks[entry_blocks.size()-1] = tail; + + // each branch in the case statement will have different breaks/continues + block_list current_breaks = breaks; + block_list current_continues = continues; + breaks.clear (); + continues.clear (); + + size_t num_incomming = 0; // number of incomming blocks to our tail + + tree_switch_case_list::iterator iter = lst->begin (); + for (size_t i = 0; i < case_blocks_num; ++iter, ++i) + { + tree_switch_case *twc = *iter; + block = entry_blocks[i]; // case_cond + assert (block); + + if (i) + blocks.push_back (entry_blocks[i]); // first block already pushed + + if (! twc->is_default_case ()) + { + // compare result of switch expression with actual case label + tree_expression *te = twc->case_label (); + jit_value *label = visit (te); + assert(label); + + const jit_operation& fn = jit_typeinfo::binary_op (octave_value::op_eq); + jit_value *cond = create_checked (fn, value, label); + assert(cond); + + jit_call *check = create_checked (&jit_typeinfo::logically_true, + cond); + + jit_block *body = factory.create<jit_block> ("case_body"); + blocks.push_back (body); + + block->append (factory.create<jit_cond_branch> (check, body, + entry_blocks[i+1])); + block = body; // case_body + } + + tree_statement_list *stmt_lst = twc->commands (); + assert(stmt_lst); + + try + { + stmt_lst->accept (*this); + num_incomming++; + block->append (factory.create<jit_branch> (tail)); + } + catch (const jit_break_exception&) + { } + + // each branch in the case statement will have different breaks/continues + current_breaks.splice (current_breaks.end (), breaks); + current_continues.splice (current_continues.end (), continues); + } + + // each branch in the case statement will have different breaks/continues + breaks.splice (breaks.end (), current_breaks); + continues.splice (continues.end (), current_continues); + + if (num_incomming || ! has_otherwise) + { + blocks.push_back (tail); + block = tail; // switch_tail + } + else + throw jit_break_exception (); // every branch broke +} + +void +jit_convert::visit_try_catch_command (tree_try_catch_command&) +{ + throw jit_fail_exception ("No visit_try_catch_command implementation"); +} + +void +jit_convert::visit_unwind_protect_command (tree_unwind_protect_command&) +{ + throw jit_fail_exception ("No visit_unwind_protect_command implementation"); +} + +void +jit_convert::visit_while_command (tree_while_command& wc) +{ + octave::unwind_protect frame; + frame.protect_var (breaks); + frame.protect_var (continues); + breaks.clear (); + continues.clear (); + + jit_block *cond_check = factory.create<jit_block> ("while_cond_check"); + block->append (factory.create<jit_branch> (cond_check)); + blocks.push_back (cond_check); + block = cond_check; + + tree_expression *expr = wc.condition (); + assert (expr && "While expression can not be null"); + jit_value *check = visit (expr); + check = create_checked (&jit_typeinfo::logically_true, check); + + jit_block *body = factory.create<jit_block> ("while_body"); + blocks.push_back (body); + + jit_block *tail = factory.create<jit_block> ("while_tail"); + block->append (factory.create<jit_cond_branch> (check, body, tail)); + block = body; + + tree_statement_list *loop_body = wc.body (); + bool all_breaking = false; + if (loop_body) + { + try + { + loop_body->accept (*this); + } + catch (const jit_break_exception&) + { + all_breaking = true; + } + } + + finish_breaks (tail, breaks); + + if (! all_breaking || continues.size ()) + { + jit_block *interrupt_check + = factory.create<jit_block> ("interrupt_check"); + blocks.push_back (interrupt_check); + finish_breaks (interrupt_check, continues); + if (! all_breaking) + block->append (factory.create<jit_branch> (interrupt_check)); + + block = interrupt_check; + jit_error_check *ec + = factory.create<jit_error_check> (jit_error_check::var_interrupt, + cond_check, final_block); + block->append (ec); + } + + blocks.push_back (tail); + block = tail; +} + +void +jit_convert::visit_do_until_command (tree_do_until_command& duc) +{ + octave::unwind_protect frame; + frame.protect_var (breaks); + frame.protect_var (continues); + breaks.clear (); + continues.clear (); + + jit_block *body = factory.create<jit_block> ("do_until_body"); + jit_block *cond_check = factory.create<jit_block> ("do_until_cond_check"); + jit_block *tail = factory.create<jit_block> ("do_until_tail"); + + block->append (factory.create<jit_branch> (body)); + blocks.push_back (body); + block = body; + + tree_statement_list *loop_body = duc.body (); + bool all_breaking = false; + if (loop_body) + { + try + { + loop_body->accept (*this); + } + catch (const jit_break_exception&) + { + all_breaking = true; + } + } + + finish_breaks (tail, breaks); + + if (! all_breaking || continues.size ()) + { + jit_block *interrupt_check + = factory.create<jit_block> ("interrupt_check"); + blocks.push_back (interrupt_check); + finish_breaks (interrupt_check, continues); + if (! all_breaking) + block->append (factory.create<jit_branch> (interrupt_check)); + + block = interrupt_check; + jit_error_check *ec + = factory.create<jit_error_check> (jit_error_check::var_interrupt, + cond_check, final_block); + block->append (ec); + + blocks.push_back (cond_check); + block = cond_check; + + tree_expression *expr = duc.condition (); + assert (expr && "Do-Until expression can not be null"); + jit_value *check = visit (expr); + check = create_checked (&jit_typeinfo::logically_true, check); + + block->append (factory.create<jit_cond_branch> (check, tail, body)); + } + + blocks.push_back (tail); + block = tail; +} + +void +jit_convert::initialize (symbol_table::scope_id s) +{ + scope = s; + iterator_count = 0; + for_bounds_count = 0; + short_count = 0; + jit_instruction::reset_ids (); + + entry_block = factory.create<jit_block> ("body"); + final_block = factory.create<jit_block> ("final"); + blocks.push_back (entry_block); + entry_block->mark_alive (); + block = entry_block; +} + +jit_call * +jit_convert::create_checked_impl (jit_call *ret) +{ + block->append (ret); + + jit_block *normal = factory.create<jit_block> (block->name ()); + jit_error_check *check + = factory.create<jit_error_check> (jit_error_check::var_error_state, ret, + normal, final_block); + block->append (check); + blocks.push_back (normal); + block = normal; + + return ret; +} + +jit_variable * +jit_convert::find_variable (const std::string& vname) const +{ + variable_map::const_iterator iter; + iter = vmap.find (vname); + return iter != vmap.end () ? iter->second : 0; +} + +jit_variable * +jit_convert::get_variable (const std::string& vname) +{ + jit_variable *ret = find_variable (vname); + if (ret) + return ret; + + symbol_table::symbol_record record = symbol_table::find_symbol (vname, scope); + if (record.is_persistent () || record.is_global ()) + throw jit_fail_exception ("Persistent and global not yet supported"); + + if (converting_function) + return create_variable (vname, jit_typeinfo::get_any (), false); + else + { + octave_value val = record.varval (); + if (val.is_undefined ()) + val = symbol_table::find_function (vname); + + jit_type *type = jit_typeinfo::type_of (val); + bounds.push_back (type_bound (type, vname)); + + return create_variable (vname, type); + } +} + +jit_variable * +jit_convert::create_variable (const std::string& vname, jit_type *type, + bool isarg) +{ + jit_variable *var = factory.create<jit_variable> (vname); + + if (isarg) + { + jit_extract_argument *extract; + extract = factory.create<jit_extract_argument> (type, var); + entry_block->prepend (extract); + } + else + { + jit_call *init = factory.create<jit_call> (&jit_typeinfo::create_undef); + jit_assign *assign = factory.create<jit_assign> (var, init); + entry_block->prepend (assign); + entry_block->prepend (init); + } + + return vmap[vname] = var; +} + +std::string +jit_convert::next_name (const char *prefix, size_t& count, bool inc) +{ + std::stringstream ss; + ss << prefix << count; + if (inc) + ++count; + return ss.str (); +} + +jit_instruction * +jit_convert::resolve (tree_index_expression& exp, jit_value *extra_arg, + bool lhs) +{ + std::string type = exp.type_tags (); + if (! (type.size () == 1 && type[0] == '(')) + throw jit_fail_exception ("Unsupported index operation"); + + std::list<tree_argument_list *> args = exp.arg_lists (); + if (args.size () != 1) + throw jit_fail_exception ("Bad number of arguments in " + "tree_index_expression"); + + tree_argument_list *arg_list = args.front (); + if (! arg_list) + throw jit_fail_exception ("null argument list"); + + if (arg_list->size () < 1) + throw jit_fail_exception ("Empty arg_list"); + + tree_expression *tree_object = exp.expression (); + jit_value *object; + if (lhs) + { + tree_identifier *id = dynamic_cast<tree_identifier *> (tree_object); + if (! id) + throw jit_fail_exception ("expected identifier"); + object = get_variable (id->name ()); + } + else + object = visit (tree_object); + + size_t narg = arg_list->size (); + tree_argument_list::iterator iter = arg_list->begin (); + bool have_extra = extra_arg; + std::vector<jit_value *> call_args (narg + 1 + have_extra); + call_args[0] = object; + + for (size_t idx = 0; iter != arg_list->end (); ++idx, ++iter) + { + octave::unwind_protect frame; + frame.add_method (&end_context, + &std::vector<jit_magic_end::context>::pop_back); + + jit_magic_end::context ctx (factory, object, idx, narg); + end_context.push_back (ctx); + call_args[idx + 1] = visit (*iter); + } + + if (extra_arg) + call_args[call_args.size () - 1] = extra_arg; + + const jit_operation& fres = lhs ? jit_typeinfo::paren_subsasgn () + : jit_typeinfo::paren_subsref (); + + return create_checked (fres, call_args); +} + +jit_value * +jit_convert::do_assign (tree_expression *exp, jit_value *rhs, bool artificial) +{ + if (! exp) + throw jit_fail_exception ("NULL lhs in assign"); + + if (isa<tree_identifier> (exp)) + return do_assign (exp->name (), rhs, exp->print_result (), artificial); + else if (tree_index_expression *idx + = dynamic_cast<tree_index_expression *> (exp)) + { + jit_value *new_object = resolve (*idx, rhs, true); + do_assign (idx->expression (), new_object, true); + + // FIXME: Will not work for values that must be release/grabed + return rhs; + } + else + throw jit_fail_exception ("Unsupported assignment"); +} + +jit_value * +jit_convert::do_assign (const std::string& lhs, jit_value *rhs, + bool print, bool artificial) +{ + jit_variable *var = get_variable (lhs); + jit_assign *assign = block->append (factory.create<jit_assign> (var, rhs)); + + if (artificial) + assign->mark_artificial (); + + if (print) + { + const jit_operation& print_fn = jit_typeinfo::print_value (); + jit_const_string *name = factory.create<jit_const_string> (lhs); + block->append (factory.create<jit_call> (print_fn, name, var)); + } + + return var; +} + +jit_value * +jit_convert::visit (tree& tee) +{ + octave::unwind_protect frame; + frame.protect_var (result); + + tee.accept (*this); + return result; +} + +void +jit_convert::finish_breaks (jit_block *dest, const block_list& lst) +{ + for (block_list::const_iterator iter = lst.begin (); iter != lst.end (); + ++iter) + { + jit_block *b = *iter; + b->append (factory.create<jit_branch> (dest)); + } +} + +// -------------------- jit_convert_llvm -------------------- +llvm::Function * +jit_convert_llvm::convert_loop (llvm::Module *module, + const jit_block_list& blocks, + const std::list<jit_value *>& constants) +{ + converting_function = false; + + // for now just init arguments from entry, later we will have to do something + // more interesting + jit_block *entry_block = blocks.front (); + for (jit_block::iterator iter = entry_block->begin (); + iter != entry_block->end (); ++iter) + if (jit_extract_argument *extract + = dynamic_cast<jit_extract_argument *> (*iter)) + argument_vec.push_back (std::make_pair (extract->name (), true)); + + jit_type *any = jit_typeinfo::get_any (); + + // argument is an array of octave_base_value*, or octave_base_value** + llvm::Type *arg_type = any->to_llvm (); // this is octave_base_value* + arg_type = arg_type->getPointerTo (); + llvm::FunctionType *ft; + ft = llvm::FunctionType::get (llvm::Type::getVoidTy (context), arg_type, + false); + function = llvm::Function::Create (ft, llvm::Function::ExternalLinkage, + "foobar", module); + + try + { + prelude = llvm::BasicBlock::Create (context, "prelude", function); + builder.SetInsertPoint (prelude); + + llvm::Value *arg = function->arg_begin (); + for (size_t i = 0; i < argument_vec.size (); ++i) + { + llvm::Value *loaded_arg = builder.CreateConstInBoundsGEP1_32 (arg, i); + arguments[argument_vec[i].first] = loaded_arg; + } + + convert (blocks, constants); + } + catch (const jit_fail_exception& e) + { + function->eraseFromParent (); + throw; + } + + return function; +} + +jit_function +jit_convert_llvm::convert_function (llvm::Module *module, + const jit_block_list& blocks, + const std::list<jit_value *>& constants, + octave_user_function& fcn, + const std::vector<jit_type *>& args) +{ + converting_function = true; + + jit_block *final_block = blocks.back (); + jit_return *ret = dynamic_cast<jit_return *> (final_block->back ()); + assert (ret); + + creating = jit_function (module, jit_convention::internal, + "foobar", ret->result_type (), args); + function = creating.to_llvm (); + + try + { + prelude = creating.new_block ("prelude"); + builder.SetInsertPoint (prelude); + + tree_parameter_list *plist = fcn.parameter_list (); + if (plist) + { + tree_parameter_list::iterator piter = plist->begin (); + tree_parameter_list::iterator pend = plist->end (); + for (size_t i = 0; i < args.size () && piter != pend; ++i, ++piter) + { + tree_decl_elt *elt = *piter; + std::string arg_name = elt->name (); + arguments[arg_name] = creating.argument (builder, i); + } + } + + convert (blocks, constants); + } + catch (const jit_fail_exception& e) + { + function->eraseFromParent (); + throw; + } + + return creating; +} + +void +jit_convert_llvm::convert (const jit_block_list& blocks, + const std::list<jit_value *>& constants) +{ + std::list<jit_block *>::const_iterator biter; + for (biter = blocks.begin (); biter != blocks.end (); ++biter) + { + jit_block *jblock = *biter; + llvm::BasicBlock *block = llvm::BasicBlock::Create (context, + jblock->name (), + function); + jblock->stash_llvm (block); + } + + jit_block *first = *blocks.begin (); + builder.CreateBr (first->to_llvm ()); + + // constants aren't in the IR, we visit those first + for (std::list<jit_value *>::const_iterator iter = constants.begin (); + iter != constants.end (); ++iter) + if (! isa<jit_instruction> (*iter)) + visit (*iter); + + // convert all instructions + for (biter = blocks.begin (); biter != blocks.end (); ++biter) + visit (*biter); + + // now finish phi nodes + for (biter = blocks.begin (); biter != blocks.end (); ++biter) + { + jit_block& block = **biter; + for (jit_block::iterator piter = block.begin (); + piter != block.end () && isa<jit_phi> (*piter); ++piter) + { + jit_instruction *phi = *piter; + finish_phi (static_cast<jit_phi *> (phi)); + } + } +} + +void +jit_convert_llvm::finish_phi (jit_phi *phi) +{ + llvm::PHINode *llvm_phi = phi->to_llvm (); + for (size_t i = 0; i < phi->argument_count (); ++i) + { + llvm::BasicBlock *pred = phi->incomming_llvm (i); + llvm_phi->addIncoming (phi->argument_llvm (i), pred); + } +} + +void +jit_convert_llvm::visit (jit_const_string& cs) +{ + cs.stash_llvm (builder.CreateGlobalStringPtr (cs.value ())); +} + +void +jit_convert_llvm::visit (jit_const_bool& cb) +{ + cb.stash_llvm (llvm::ConstantInt::get (cb.type_llvm (), cb.value ())); +} + +void +jit_convert_llvm::visit (jit_const_scalar& cs) +{ + cs.stash_llvm (llvm::ConstantFP::get (cs.type_llvm (), cs.value ())); +} + +void +jit_convert_llvm::visit (jit_const_complex& cc) +{ + llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); + Complex value = cc.value (); + llvm::Value *real = llvm::ConstantFP::get (scalar_t, value.real ()); + llvm::Value *imag = llvm::ConstantFP::get (scalar_t, value.imag ()); + cc.stash_llvm (jit_typeinfo::create_complex (real, imag)); +} + +void jit_convert_llvm::visit (jit_const_index& ci) +{ + ci.stash_llvm (llvm::ConstantInt::get (ci.type_llvm (), ci.value ())); +} + +void +jit_convert_llvm::visit (jit_const_range& cr) +{ + llvm::StructType *stype = llvm::cast<llvm::StructType>(cr.type_llvm ()); + llvm::Type *scalar_t = jit_typeinfo::get_scalar_llvm (); + llvm::Type *idx = jit_typeinfo::get_index_llvm (); + const jit_range& rng = cr.value (); + + llvm::Constant *constants[4]; + constants[0] = llvm::ConstantFP::get (scalar_t, rng.base); + constants[1] = llvm::ConstantFP::get (scalar_t, rng.limit); + constants[2] = llvm::ConstantFP::get (scalar_t, rng.inc); + constants[3] = llvm::ConstantInt::get (idx, rng.nelem); + + llvm::Value *as_llvm; + as_llvm = llvm::ConstantStruct::get (stype, + llvm::makeArrayRef (constants, 4)); + cr.stash_llvm (as_llvm); +} + +void +jit_convert_llvm::visit (jit_block& b) +{ + llvm::BasicBlock *block = b.to_llvm (); + builder.SetInsertPoint (block); + for (jit_block::iterator iter = b.begin (); iter != b.end (); ++iter) + visit (*iter); +} + +void +jit_convert_llvm::visit (jit_branch& b) +{ + b.stash_llvm (builder.CreateBr (b.successor_llvm ())); +} + +void +jit_convert_llvm::visit (jit_cond_branch& cb) +{ + llvm::Value *cond = cb.cond_llvm (); + llvm::Value *br; + br = builder.CreateCondBr (cond, cb.successor_llvm (0), + cb.successor_llvm (1)); + cb.stash_llvm (br); +} + +void +jit_convert_llvm::visit (jit_call& call) +{ + const jit_function& ol = call.overload (); + + std::vector<jit_value *> args (call.arguments ().size ()); + for (size_t i = 0; i < args.size (); ++i) + args[i] = call.argument (i); + + llvm::Value *ret = ol.call (builder, args); + call.stash_llvm (ret); +} + +void +jit_convert_llvm::visit (jit_extract_argument& extract) +{ + llvm::Value *arg = arguments[extract.name ()]; + assert (arg); + + if (converting_function) + extract.stash_llvm (arg); + else + { + arg = builder.CreateLoad (arg); + + const jit_function& ol = extract.overload (); + extract.stash_llvm (ol.call (builder, arg)); + } +} + +void +jit_convert_llvm::visit (jit_store_argument& store) +{ + const jit_function& ol = store.overload (); + llvm::Value *arg_value = ol.call (builder, store.result ()); + llvm::Value *arg = arguments[store.name ()]; + store.stash_llvm (builder.CreateStore (arg_value, arg)); +} + +void +jit_convert_llvm::visit (jit_return& ret) +{ + jit_value *res = ret.result (); + + if (converting_function) + creating.do_return (builder, res->to_llvm (), false); + else + { + if (res) + builder.CreateRet (res->to_llvm ()); + else + builder.CreateRetVoid (); + } +} + +void +jit_convert_llvm::visit (jit_phi& phi) +{ + // we might not have converted all incoming branches, so we don't + // set incomming branches now + llvm::PHINode *node = llvm::PHINode::Create (phi.type_llvm (), + phi.argument_count ()); + builder.Insert (node); + phi.stash_llvm (node); +} + +void +jit_convert_llvm::visit (jit_variable&) +{ + throw jit_fail_exception ("ERROR: SSA construction should remove all variables"); +} + +void +jit_convert_llvm::visit (jit_error_check& check) +{ + llvm::Value *cond; + + switch (check.check_variable ()) + { + case jit_error_check::var_error_state: + cond = jit_typeinfo::insert_error_check (builder); + break; + case jit_error_check::var_interrupt: + cond = jit_typeinfo::insert_interrupt_check (builder); + break; + default: + panic_impossible (); + } + + llvm::Value *br = builder.CreateCondBr (cond, check.successor_llvm (0), + check.successor_llvm (1)); + check.stash_llvm (br); +} + +void +jit_convert_llvm::visit (jit_assign& assign) +{ + jit_value *new_value = assign.src (); + assign.stash_llvm (new_value->to_llvm ()); + + if (assign.artificial ()) + return; + + jit_value *overwrite = assign.overwrite (); + if (isa<jit_assign_base> (overwrite)) + { + const jit_function& ol = jit_typeinfo::get_release (overwrite->type ()); + if (ol.valid ()) + ol.call (builder, overwrite); + } +} + +void +jit_convert_llvm::visit (jit_argument&) +{ } + +void +jit_convert_llvm::visit (jit_magic_end& me) +{ + const jit_function& ol = me.overload (); + + jit_magic_end::context ctx = me.resolve_context (); + llvm::Value *ret = ol.call (builder, ctx.value, ctx.index, ctx.count); + me.stash_llvm (ret); +} + +// -------------------- jit_infer -------------------- +jit_infer::jit_infer (jit_factory& afactory, jit_block_list& ablocks, + const variable_map& avmap) + : blocks (ablocks), factory (afactory), vmap (avmap) { } + +void +jit_infer::infer (void) +{ + construct_ssa (); + + // initialize the worklist to instructions derived from constants + const std::list<jit_value *>& constants = factory.constants (); + for (std::list<jit_value *>::const_iterator iter = constants.begin (); + iter != constants.end (); ++iter) + append_users (*iter); + + // the entry block terminator may be a regular branch statement + if (entry_block ().terminator ()) + push_worklist (entry_block ().terminator ()); + + // FIXME: Describe algorithm here + while (worklist.size ()) + { + jit_instruction *next = worklist.front (); + worklist.pop_front (); + next->stash_in_worklist (false); + + if (next->infer ()) + { + // terminators need to be handles specially + if (jit_terminator *term = dynamic_cast<jit_terminator *> (next)) + append_users_term (term); + else + append_users (next); + } + } + + remove_dead (); + blocks.label (); + place_releases (); + simplify_phi (); +} + +void +jit_infer::append_users (jit_value *v) +{ + for (jit_use *use = v->first_use (); use; use = use->next ()) + push_worklist (use->user ()); +} + +void +jit_infer::append_users_term (jit_terminator *term) +{ + for (size_t i = 0; i < term->successor_count (); ++i) + { + if (term->alive (i)) + { + jit_block *succ = term->successor (i); + for (jit_block::iterator iter = succ->begin (); + iter != succ->end () && isa<jit_phi> (*iter); ++iter) + push_worklist (*iter); + + jit_terminator *sterm = succ->terminator (); + if (sterm) + push_worklist (sterm); + } + } +} + +void +jit_infer::construct_ssa (void) +{ + blocks.label (); + final_block ().compute_idom (entry_block ()); + entry_block ().compute_df (); + entry_block ().create_dom_tree (); + + // insert phi nodes where needed, this is done on a per variable basis + for (variable_map::const_iterator iter = vmap.begin (); iter != vmap.end (); + ++iter) + { + jit_block::df_set visited, added_phi; + std::list<jit_block *> ssa_worklist; + iter->second->use_blocks (visited); + ssa_worklist.insert (ssa_worklist.begin (), visited.begin (), + visited.end ()); + + while (ssa_worklist.size ()) + { + jit_block *b = ssa_worklist.front (); + ssa_worklist.pop_front (); + + for (jit_block::df_iterator diter = b->df_begin (); + diter != b->df_end (); ++diter) + { + jit_block *dblock = *diter; + if (! added_phi.count (dblock)) + { + jit_phi *phi = factory.create<jit_phi> (iter->second, + dblock->use_count ()); + dblock->prepend (phi); + added_phi.insert (dblock); + } + + if (! visited.count (dblock)) + { + ssa_worklist.push_back (dblock); + visited.insert (dblock); + } + } + } + } + + do_construct_ssa (entry_block (), entry_block ().visit_count ()); +} + +void +jit_infer::do_construct_ssa (jit_block& ablock, size_t avisit_count) +{ + if (ablock.visited (avisit_count)) + return; + + // replace variables with their current SSA value + for (jit_block::iterator iter = ablock.begin (); iter != ablock.end (); + ++iter) + { + jit_instruction *instr = *iter; + instr->construct_ssa (); + instr->push_variable (); + } + + // finish phi nodes of successors + for (size_t i = 0; i < ablock.successor_count (); ++i) + { + jit_block *finish = ablock.successor (i); + + for (jit_block::iterator iter = finish->begin (); + iter != finish->end () && isa<jit_phi> (*iter);) + { + jit_phi *phi = static_cast<jit_phi *> (*iter); + jit_variable *var = phi->dest (); + ++iter; + + if (var->has_top ()) + phi->add_incomming (&ablock, var->top ()); + else + { + // temporaries may have extranious phi nodes which can be removed + assert (! phi->use_count ()); + assert (var->name ().size () && var->name ()[0] == '#'); + phi->remove (); + } + } + } + + for (size_t i = 0; i < ablock.dom_successor_count (); ++i) + do_construct_ssa (*ablock.dom_successor (i), avisit_count); + + ablock.pop_all (); +} + +void +jit_infer::place_releases (void) +{ + std::set<jit_value *> temporaries; + for (jit_block_list::iterator iter = blocks.begin (); iter != blocks.end (); + ++iter) + { + jit_block& ablock = **iter; + if (ablock.id () != jit_block::NO_ID) + { + release_temp (ablock, temporaries); + release_dead_phi (ablock); + } + } +} + +void +jit_infer::push_worklist (jit_instruction *instr) +{ + if (! instr->in_worklist ()) + { + instr->stash_in_worklist (true); + worklist.push_back (instr); + } +} + +void +jit_infer::remove_dead () +{ + jit_block_list::iterator biter; + for (biter = blocks.begin (); biter != blocks.end (); ++biter) + { + jit_block *b = *biter; + if (b->alive ()) + { + for (jit_block::iterator iter = b->begin (); + iter != b->end () && isa<jit_phi> (*iter);) + { + jit_phi *phi = static_cast<jit_phi *> (*iter); + if (phi->prune ()) + iter = b->remove (iter); + else + ++iter; + } + } + } + + for (biter = blocks.begin (); biter != blocks.end ();) + { + jit_block *b = *biter; + if (b->alive ()) + { + // FIXME: A special case for jit_error_check, if we generalize to + // we will need to change! + jit_terminator *term = b->terminator (); + if (term && term->successor_count () == 2 && ! term->alive (0)) + { + jit_block *succ = term->successor (1); + term->remove (); + jit_branch *abreak = factory.create<jit_branch> (succ); + b->append (abreak); + abreak->infer (); + } + + ++biter; + } + else + { + jit_terminator *term = b->terminator (); + if (term) + term->remove (); + biter = blocks.erase (biter); + } + } +} + +void +jit_infer::release_dead_phi (jit_block& ablock) +{ + jit_block::iterator iter = ablock.begin (); + while (iter != ablock.end () && isa<jit_phi> (*iter)) + { + jit_phi *phi = static_cast<jit_phi *> (*iter); + ++iter; + + jit_use *use = phi->first_use (); + if (phi->use_count () == 1 && isa<jit_assign> (use->user ())) + { + // instead of releasing on assign, release on all incomming branches, + // this can get rid of casts inside loops + for (size_t i = 0; i < phi->argument_count (); ++i) + { + jit_value *arg = phi->argument (i); + if (! arg->needs_release ()) + continue; + + jit_block *inc = phi->incomming (i); + jit_block *split = inc->maybe_split (factory, blocks, ablock); + jit_terminator *term = split->terminator (); + jit_call *release + = factory.create<jit_call> (jit_typeinfo::release, arg); + release->infer (); + split->insert_before (term, release); + } + + phi->replace_with (0); + phi->remove (); + } + } +} + +void +jit_infer::release_temp (jit_block& ablock, std::set<jit_value *>& temp) +{ + for (jit_block::iterator iter = ablock.begin (); iter != ablock.end (); + ++iter) + { + jit_instruction *instr = *iter; + + // check for temporaries that require release and live across + // multiple blocks + if (instr->needs_release ()) + { + jit_block *fu_block = instr->first_use_block (); + if (fu_block && fu_block != &ablock && instr->needs_release ()) + temp.insert (instr); + } + + if (isa<jit_call> (instr)) + { + // place releases for temporary arguments + for (size_t i = 0; i < instr->argument_count (); ++i) + { + jit_value *arg = instr->argument (i); + if (! arg->needs_release ()) + continue; + + jit_call *release + = factory.create<jit_call> (&jit_typeinfo::release, arg); + release->infer (); + ablock.insert_after (iter, release); + ++iter; + temp.erase (arg); + } + } + } + + if (! temp.size () || ! isa<jit_error_check> (ablock.terminator ())) + return; + + // FIXME: If we support try/catch or unwind_protect final_block + // may not be the destination + jit_block *split = ablock.maybe_split (factory, blocks, final_block ()); + jit_terminator *term = split->terminator (); + for (std::set<jit_value *>::const_iterator iter = temp.begin (); + iter != temp.end (); ++iter) + { + jit_value *value = *iter; + jit_call *release + = factory.create<jit_call> (&jit_typeinfo::release, value); + split->insert_before (term, release); + release->infer (); + } +} + +void +jit_infer::simplify_phi (void) +{ + for (jit_block_list::iterator biter = blocks.begin (); biter != blocks.end (); + ++biter) + { + jit_block &ablock = **biter; + for (jit_block::iterator iter = ablock.begin (); + iter != ablock.end () && isa<jit_phi> (*iter); ++iter) + simplify_phi (*static_cast<jit_phi *> (*iter)); + } +} + +void +jit_infer::simplify_phi (jit_phi& phi) +{ + jit_block& pblock = *phi.parent (); + const jit_operation& cast_fn = jit_typeinfo::cast (phi.type ()); + jit_variable *dest = phi.dest (); + for (size_t i = 0; i < phi.argument_count (); ++i) + { + jit_value *arg = phi.argument (i); + if (arg->type () != phi.type ()) + { + jit_block *pred = phi.incomming (i); + jit_block *split = pred->maybe_split (factory, blocks, pblock); + jit_terminator *term = split->terminator (); + jit_instruction *cast = factory.create<jit_call> (cast_fn, arg); + jit_assign *assign = factory.create<jit_assign> (dest, cast); + + split->insert_before (term, cast); + split->insert_before (term, assign); + cast->infer (); + assign->infer (); + phi.stash_argument (i, assign); + } + } +} + +// -------------------- tree_jit -------------------- + +tree_jit::tree_jit (void) : module (0), engine (0) +{ } + +tree_jit::~tree_jit (void) +{ } + +bool +tree_jit::execute (tree_simple_for_command& cmd, const octave_value& bounds) +{ + return instance ().do_execute (cmd, bounds); +} + +bool +tree_jit::execute (tree_while_command& cmd) +{ + return instance ().do_execute (cmd); +} + +bool +tree_jit::execute (octave_user_function& fcn, const octave_value_list& args, + octave_value_list& retval) +{ + return instance ().do_execute (fcn, args, retval); +} + +tree_jit& +tree_jit::instance (void) +{ + static tree_jit ret; + return ret; +} + +bool +tree_jit::initialize (void) +{ + if (engine) + return true; + + if (! module) + { + llvm::InitializeNativeTarget (); + module = new llvm::Module ("octave", context); + } + + // sometimes this fails pre main + engine = llvm::ExecutionEngine::createJIT (module); + + if (! engine) + return false; + +#if defined (LEGACY_PASSMANAGER) + module_pass_manager = new llvm::legacy::PassManager (); + pass_manager = new llvm::legacy::FunctionPassManager (module); +#else + module_pass_manager = new llvm::PassManager (); + pass_manager = new llvm::FunctionPassManager (module); +#endif + module_pass_manager->add (llvm::createAlwaysInlinerPass ()); + +#if defined (HAVE_LLVM_DATALAYOUT) + pass_manager->add (new llvm::DataLayout (*engine->getDataLayout ())); +#else + pass_manager->add (new llvm::TargetData (*engine->getTargetData ())); +#endif + pass_manager->add (llvm::createCFGSimplificationPass ()); + pass_manager->add (llvm::createBasicAliasAnalysisPass ()); + pass_manager->add (llvm::createPromoteMemoryToRegisterPass ()); + pass_manager->add (llvm::createInstructionCombiningPass ()); + pass_manager->add (llvm::createReassociatePass ()); + pass_manager->add (llvm::createGVNPass ()); + pass_manager->add (llvm::createCFGSimplificationPass ()); + pass_manager->doInitialization (); + + jit_typeinfo::initialize (module, engine); + + return true; +} + +bool +tree_jit::do_execute (tree_simple_for_command& cmd, const octave_value& bounds) +{ + size_t tc = trip_count (bounds); + if (! tc || ! initialize () || ! enabled ()) + return false; + + jit_info::vmap extra_vars; + extra_vars["#for_bounds0"] = &bounds; + + jit_info *info = cmd.get_info (); + if (! info || ! info->match (extra_vars)) + { + if (tc < static_cast<size_t> (Vjit_startcnt)) + return false; + + delete info; + info = new jit_info (*this, cmd, bounds); + cmd.stash_info (info); + } + + return info->execute (extra_vars); +} + +bool +tree_jit::do_execute (tree_while_command& cmd) +{ + if (! initialize () || ! enabled ()) + return false; + + jit_info *info = cmd.get_info (); + if (! info || ! info->match ()) + { + delete info; + info = new jit_info (*this, cmd); + cmd.stash_info (info); + } + + return info->execute (); +} + +bool +tree_jit::do_execute (octave_user_function& fcn, const octave_value_list& args, + octave_value_list& retval) +{ + if (! initialize () || ! enabled ()) + return false; + + jit_function_info *info = fcn.get_info (); + if (! info || ! info->match (args)) + { + delete info; + info = new jit_function_info (*this, fcn, args); + fcn.stash_info (info); + } + + return info->execute (args, retval); +} + +bool +tree_jit::enabled (void) +{ + // Ideally, we should only disable JIT if there is a breakpoint in the code + // we are about to run. However, we can't figure this out in O(1) time, so + // we conservatively check for the existence of any breakpoints. + return (Vjit_enable && ! bp_table::have_breakpoints () + && ! octave::Vdebug_on_interrupt && ! Vdebug_on_error); +} + +size_t +tree_jit::trip_count (const octave_value& bounds) const +{ + if (bounds.is_range ()) + { + Range rng = bounds.range_value (); + return rng.numel (); + } + + // unsupported type + return 0; +} + +void +tree_jit::optimize (llvm::Function *fn) +{ + if (Vdebug_jit) + llvm::verifyModule (*module); + + module_pass_manager->run (*module); + pass_manager->run (*fn); + + if (Vdebug_jit) + { + std::string error; +#if defined (RAW_FD_OSTREAM_ARG_IS_LLVM_SYS_FS) + llvm::raw_fd_ostream fout ("test.bc", error, + llvm::sys::fs::F_Binary); +#else + llvm::raw_fd_ostream fout ("test.bc", error, + llvm::raw_fd_ostream::F_Binary); +#endif + llvm::WriteBitcodeToFile (module, fout); + } +} + +// -------------------- jit_function_info -------------------- +jit_function_info::jit_function_info (tree_jit& tjit, + octave_user_function& fcn, + const octave_value_list& ov_args) + : argument_types (ov_args.length ()), function (0) +{ + size_t nargs = ov_args.length (); + for (size_t i = 0; i < nargs; ++i) + argument_types[i] = jit_typeinfo::type_of (ov_args(i)); + + jit_function raw_fn; + jit_function wrapper; + + try + { + jit_convert conv (fcn, argument_types); + jit_infer infer (conv.get_factory (), conv.get_blocks (), + conv.get_variable_map ()); + infer.infer (); + + if (Vdebug_jit) + { + jit_block_list& blocks = infer.get_blocks (); + blocks.label (); + std::cout << "-------------------- Compiling function "; + std::cout << "--------------------\n"; + + tree_print_code tpc (std::cout); + tpc.visit_octave_user_function_header (fcn); + tpc.visit_statement_list (*fcn.body ()); + tpc.visit_octave_user_function_trailer (fcn); + blocks.print (std::cout, "octave jit ir"); + } + + jit_factory& factory = conv.get_factory (); + llvm::Module *module = tjit.get_module (); + jit_convert_llvm to_llvm; + raw_fn = to_llvm.convert_function (module, infer.get_blocks (), + factory.constants (), fcn, + argument_types); + + if (Vdebug_jit) + { + std::cout << "-------------------- raw function "; + std::cout << "--------------------\n"; + std::cout << *raw_fn.to_llvm () << std::endl; + llvm::verifyFunction (*raw_fn.to_llvm ()); + } + + std::string wrapper_name = fcn.name () + "_wrapper"; + jit_type *any_t = jit_typeinfo::get_any (); + std::vector<jit_type *> wrapper_args (1, jit_typeinfo::get_any_ptr ()); + wrapper = jit_function (module, jit_convention::internal, wrapper_name, + any_t, wrapper_args); + + llvm::BasicBlock *wrapper_body = wrapper.new_block (); + builder.SetInsertPoint (wrapper_body); + + llvm::Value *wrapper_arg = wrapper.argument (builder, 0); + std::vector<llvm::Value *> raw_args (nargs); + for (size_t i = 0; i < nargs; ++i) + { + llvm::Value *arg; + arg = builder.CreateConstInBoundsGEP1_32 (wrapper_arg, i); + arg = builder.CreateLoad (arg); + + jit_type *arg_type = argument_types[i]; + const jit_function& cast = jit_typeinfo::cast (arg_type, any_t); + raw_args[i] = cast.call (builder, arg); + } + + llvm::Value *result = raw_fn.call (builder, raw_args); + if (raw_fn.result ()) + { + jit_type *raw_result_t = raw_fn.result (); + const jit_function& cast = jit_typeinfo::cast (any_t, raw_result_t); + result = cast.call (builder, result); + } + else + { + llvm::Value *zero = builder.getInt32 (0); + result = builder.CreateBitCast (zero, any_t->to_llvm ()); + } + + wrapper.do_return (builder, result); + + llvm::Function *llvm_function = wrapper.to_llvm (); + tjit.optimize (llvm_function); + + if (Vdebug_jit) + { + std::cout << "-------------------- optimized and wrapped "; + std::cout << "--------------------\n"; + std::cout << *llvm_function << std::endl; + llvm::verifyFunction (*llvm_function); + } + + llvm::ExecutionEngine* engine = tjit.get_engine (); + void *void_fn = engine->getPointerToFunction (llvm_function); + function = reinterpret_cast<jited_function> (void_fn); + } + catch (const jit_fail_exception& e) + { + argument_types.clear (); + + if (Vdebug_jit) + { + if (e.known ()) + std::cout << "jit fail: " << e.what () << std::endl; + } + + Vjit_failcnt++; + + wrapper.erase (); + raw_fn.erase (); + } +} + +bool +jit_function_info::execute (const octave_value_list& ov_args, + octave_value_list& retval) const +{ + if (! function) + return false; + + // FIXME: figure out a way to delete ov_args so we avoid duplicating refcount + size_t nargs = ov_args.length (); + std::vector<octave_base_value *> args (nargs); + for (size_t i = 0; i < nargs; ++i) + { + octave_base_value *obv = ov_args(i).internal_rep (); + obv->grab (); + args[i] = obv; + } + + octave_base_value *ret = function (&args[0]); + if (ret) + retval(0) = octave_value (ret); + + octave_quit (); + + return true; +} + +bool +jit_function_info::match (const octave_value_list& ov_args) const +{ + if (! function) + return true; + + size_t nargs = ov_args.length (); + if (nargs != argument_types.size ()) + return false; + + for (size_t i = 0; i < nargs; ++i) + if (jit_typeinfo::type_of (ov_args(i)) != argument_types[i]) + return false; + + return true; +} + +// -------------------- jit_info -------------------- +jit_info::jit_info (tree_jit& tjit, tree& tee) + : engine (tjit.get_engine ()), function (0), llvm_function (0) +{ + compile (tjit, tee); +} + +jit_info::jit_info (tree_jit& tjit, tree& tee, const octave_value& for_bounds) + : engine (tjit.get_engine ()), function (0), llvm_function (0) +{ + compile (tjit, tee, jit_typeinfo::type_of (for_bounds)); +} + +jit_info::~jit_info (void) +{ + if (llvm_function) + llvm_function->eraseFromParent (); +} + +bool +jit_info::execute (const vmap& extra_vars) const +{ + if (! function) + return false; + + std::vector<octave_base_value *> real_arguments (arguments.size ()); + for (size_t i = 0; i < arguments.size (); ++i) + { + if (arguments[i].second) + { + octave_value current = find (extra_vars, arguments[i].first); + octave_base_value *obv = current.internal_rep (); + obv->grab (); + real_arguments[i] = obv; + } + } + + function (&real_arguments[0]); + + for (size_t i = 0; i < arguments.size (); ++i) + { + const std::string& name = arguments[i].first; + + // do not store for loop bounds temporary + if (name.size () && name[0] != '#') + symbol_table::assign (arguments[i].first, real_arguments[i]); + } + + octave_quit (); + + return true; +} + +bool +jit_info::match (const vmap& extra_vars) const +{ + if (! function) + return true; + + for (size_t i = 0; i < bounds.size (); ++i) + { + const std::string& arg_name = bounds[i].second; + octave_value value = find (extra_vars, arg_name); + jit_type *type = jit_typeinfo::type_of (value); + + // FIXME: Check for a parent relationship + if (type != bounds[i].first) + return false; + } + + return true; +} + +void +jit_info::compile (tree_jit& tjit, tree& tee, jit_type *for_bounds) +{ + try + { + jit_convert conv (tee, for_bounds); + jit_infer infer (conv.get_factory (), conv.get_blocks (), + conv.get_variable_map ()); + + infer.infer (); + + if (Vdebug_jit) + { + jit_block_list& blocks = infer.get_blocks (); + blocks.label (); + std::cout << "-------------------- Compiling tree --------------------\n"; + std::cout << tee.str_print_code () << std::endl; + blocks.print (std::cout, "octave jit ir"); + } + + jit_factory& factory = conv.get_factory (); + jit_convert_llvm to_llvm; + llvm_function = to_llvm.convert_loop (tjit.get_module (), + infer.get_blocks (), + factory.constants ()); + arguments = to_llvm.get_arguments (); + bounds = conv.get_bounds (); + } + catch (const jit_fail_exception& e) + { + if (Vdebug_jit) + { + if (e.known ()) + std::cout << "jit fail: " << e.what () << std::endl; + } + + Vjit_failcnt++; + + } + + if (llvm_function) + { + if (Vdebug_jit) + { + std::cout << "-------------------- llvm ir --------------------"; + std::cout << *llvm_function << std::endl; + llvm::verifyFunction (*llvm_function); + } + + tjit.optimize (llvm_function); + + if (Vdebug_jit) + { + std::cout << "-------------------- optimized llvm ir " + << "--------------------\n"; + std::cout << *llvm_function << std::endl; + } + + void *void_fn = engine->getPointerToFunction (llvm_function); + function = reinterpret_cast<jited_function> (void_fn); + } +} + +octave_value +jit_info::find (const vmap& extra_vars, const std::string& vname) const +{ + vmap::const_iterator iter = extra_vars.find (vname); + return iter == extra_vars.end () ? symbol_table::varval (vname) + : *iter->second; +} + +#endif + +DEFUN (jit_failcnt, args, nargout, + doc: /* -*- texinfo -*- +@deftypefn {} {@var{val} =} jit_failcnt () +@deftypefnx {} {@var{old_val} =} jit_failcnt (@var{new_val}) +@deftypefnx {} {} jit_failcnt (@var{new_val}, "local") +Query or set the internal variable that counts the number of JIT fail +exceptions for Octave's JIT compiler. + +When called from inside a function with the @qcode{"local"} option, the +variable is changed locally for the function and any subroutines it calls. +The original variable value is restored when exiting the function. +@seealso{jit_enable, jit_startcnt, debug_jit} +@end deftypefn */) +{ +#if defined (HAVE_LLVM) + return SET_INTERNAL_VARIABLE (jit_failcnt); +#else + octave_unused_parameter (args); + octave_unused_parameter (nargout); + warn_disabled_feature ("jit_failcnt", "JIT compiling"); + return ovl (); +#endif +} + +DEFUN (debug_jit, args, nargout, + doc: /* -*- texinfo -*- +@deftypefn {} {@var{val} =} debug_jit () +@deftypefnx {} {@var{old_val} =} debug_jit (@var{new_val}) +@deftypefnx {} {} debug_jit (@var{new_val}, "local") +Query or set the internal variable that determines whether +debugging/tracing is enabled for Octave's JIT compiler. + +When called from inside a function with the @qcode{"local"} option, the +variable is changed locally for the function and any subroutines it calls. +The original variable value is restored when exiting the function. +@seealso{jit_enable, jit_startcnt} +@end deftypefn */) +{ +#if defined (HAVE_LLVM) + return SET_INTERNAL_VARIABLE (debug_jit); +#else + octave_unused_parameter (args); + octave_unused_parameter (nargout); + warn_disabled_feature ("debug_jit", "JIT"); + return ovl (); +#endif +} + +DEFUN (jit_enable, args, nargout, + doc: /* -*- texinfo -*- +@deftypefn {} {@var{val} =} jit_enable () +@deftypefnx {} {@var{old_val} =} jit_enable (@var{new_val}) +@deftypefnx {} {} jit_enable (@var{new_val}, "local") +Query or set the internal variable that enables Octave's JIT compiler. + +When called from inside a function with the @qcode{"local"} option, the +variable is changed locally for the function and any subroutines it calls. +The original variable value is restored when exiting the function. +@seealso{jit_startcnt, debug_jit} +@end deftypefn */) +{ +#if defined (HAVE_LLVM) + return SET_INTERNAL_VARIABLE (jit_enable); +#else + octave_unused_parameter (args); + octave_unused_parameter (nargout); + warn_disabled_feature ("jit_enable", "JIT"); + return ovl (); +#endif +} + +DEFUN (jit_startcnt, args, nargout, + doc: /* -*- texinfo -*- +@deftypefn {} {@var{val} =} jit_startcnt () +@deftypefnx {} {@var{old_val} =} jit_startcnt (@var{new_val}) +@deftypefnx {} {} jit_startcnt (@var{new_val}, "local") +Query or set the internal variable that determines whether JIT compilation +will take place for a specific loop. + +Because compilation is a costly operation it does not make sense to employ +JIT when the loop count is low. By default only loops with greater than +1000 iterations will be accelerated. + +When called from inside a function with the @qcode{"local"} option, the +variable is changed locally for the function and any subroutines it calls. +The original variable value is restored when exiting the function. +@seealso{jit_enable, jit_failcnt, debug_jit} +@end deftypefn */) +{ +#if defined (HAVE_LLVM) + return SET_INTERNAL_VARIABLE_WITH_LIMITS (jit_startcnt, 1, + std::numeric_limits<int>::max ()); +#else + octave_unused_parameter (args); + octave_unused_parameter (nargout); + warn_disabled_feature ("jit_enable", "JIT"); + return ovl (); +#endif +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libinterp/parse-tree/pt-jit.h Tue Jan 31 17:43:58 2017 -0500 @@ -0,0 +1,453 @@ +/* + +Copyright (C) 2012-2016 Max Brister + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +Octave is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, see +<http://www.gnu.org/licenses/>. + +*/ + +// Author: Max Brister <max@2bass.com> + +#if ! defined (octave_pt_jit_h) +#define octave_pt_jit_h 1 + +#include "octave-config.h" + +#if defined (HAVE_LLVM) + +#include "jit-ir.h" +#include "pt-walk.h" +#include "symtab.h" + +class octave_value_list; + +// Convert from the parse tree (AST) to the low level Octave IR. +class +jit_convert : public tree_walker +{ +public: + typedef std::pair<jit_type *, std::string> type_bound; + typedef std::vector<type_bound> type_bound_vector; + typedef std::map<std::string, jit_variable *> variable_map; + + jit_convert (tree &tee, jit_type *for_bounds = 0); + + jit_convert (octave_user_function& fcn, const std::vector<jit_type *>& args); + +#define DECL_ARG(n) const ARG ## n& arg ## n +#define JIT_CREATE_CHECKED(N) \ + template <OCT_MAKE_DECL_LIST (typename, ARG, N)> \ + jit_call *create_checked (OCT_MAKE_LIST (DECL_ARG, N)) \ + { \ + jit_call *ret = factory.create<jit_call> (OCT_MAKE_ARG_LIST (arg, N)); \ + return create_checked_impl (ret); \ + } + + JIT_CREATE_CHECKED (1) + JIT_CREATE_CHECKED (2) + JIT_CREATE_CHECKED (3) + JIT_CREATE_CHECKED (4) + +#undef JIT_CREATE_CHECKED +#undef DECL_ARG + + jit_block_list& get_blocks (void) { return blocks; } + + const type_bound_vector& get_bounds (void) const { return bounds; } + + jit_factory& get_factory (void) { return factory; } + + llvm::Function *get_function (void) const { return function; } + + const variable_map &get_variable_map (void) const { return vmap; } + + void visit_anon_fcn_handle (tree_anon_fcn_handle&); + + void visit_argument_list (tree_argument_list&); + + void visit_binary_expression (tree_binary_expression&); + + void visit_break_command (tree_break_command&); + + void visit_colon_expression (tree_colon_expression&); + + void visit_continue_command (tree_continue_command&); + + void visit_global_command (tree_global_command&); + + void visit_persistent_command (tree_persistent_command&); + + void visit_decl_elt (tree_decl_elt&); + + void visit_decl_init_list (tree_decl_init_list&); + + void visit_simple_for_command (tree_simple_for_command&); + + void visit_complex_for_command (tree_complex_for_command&); + + void visit_octave_user_script (octave_user_script&); + + void visit_octave_user_function (octave_user_function&); + + void visit_octave_user_function_header (octave_user_function&); + + void visit_octave_user_function_trailer (octave_user_function&); + + void visit_function_def (tree_function_def&); + + void visit_identifier (tree_identifier&); + + void visit_if_clause (tree_if_clause&); + + void visit_if_command (tree_if_command&); + + void visit_if_command_list (tree_if_command_list&); + + void visit_index_expression (tree_index_expression&); + + void visit_matrix (tree_matrix&); + + void visit_cell (tree_cell&); + + void visit_multi_assignment (tree_multi_assignment&); + + void visit_no_op_command (tree_no_op_command&); + + void visit_constant (tree_constant&); + + void visit_fcn_handle (tree_fcn_handle&); + + void visit_funcall (tree_funcall&); + + void visit_parameter_list (tree_parameter_list&); + + void visit_postfix_expression (tree_postfix_expression&); + + void visit_prefix_expression (tree_prefix_expression&); + + void visit_return_command (tree_return_command&); + + void visit_return_list (tree_return_list&); + + void visit_simple_assignment (tree_simple_assignment&); + + void visit_statement (tree_statement&); + + void visit_statement_list (tree_statement_list&); + + void visit_switch_case (tree_switch_case&); + + void visit_switch_case_list (tree_switch_case_list&); + + void visit_switch_command (tree_switch_command&); + + void visit_try_catch_command (tree_try_catch_command&); + + void visit_unwind_protect_command (tree_unwind_protect_command&); + + void visit_while_command (tree_while_command&); + + void visit_do_until_command (tree_do_until_command&); +private: + std::vector<std::pair<std::string, bool> > arguments; + type_bound_vector bounds; + + bool converting_function; + + // the scope of the function we are converting, or the current scope + symbol_table::scope_id scope; + + jit_factory factory; + + // used instead of return values from visit_* functions + jit_value *result; + + jit_block *entry_block; + + jit_block *final_block; + + jit_block *block; + + llvm::Function *function; + + jit_block_list blocks; + + std::vector<jit_magic_end::context> end_context; + + size_t iterator_count; + size_t for_bounds_count; + size_t short_count; + + variable_map vmap; + + void initialize (symbol_table::scope_id s); + + jit_call *create_checked_impl (jit_call *ret); + + // get an existing vairable. If the variable does not exist, it will not be + // created + jit_variable *find_variable (const std::string& vname) const; + + // get a variable, create it if it does not exist. The type will default to + // the variable's current type in the symbol table. + jit_variable *get_variable (const std::string& vname); + + // create a variable of the given name and given type. Will also insert an + // extract statement + jit_variable *create_variable (const std::string& vname, jit_type *type, + bool isarg = true); + + // The name of the next for loop iterator. If inc is false, then the + // iterator counter will not be incremented. + std::string next_iterator (bool inc = true) + { return next_name ("#iter", iterator_count, inc); } + + std::string next_for_bounds (bool inc = true) + { return next_name ("#for_bounds", for_bounds_count, inc); } + + std::string next_shortcircut_result (bool inc = true) + { return next_name ("#shortcircut_result", short_count, inc); } + + std::string next_name (const char *prefix, size_t& count, bool inc); + + jit_instruction *resolve (tree_index_expression& exp, + jit_value *extra_arg = 0, bool lhs = false); + + jit_value *do_assign (tree_expression *exp, jit_value *rhs, + bool artificial = false); + + jit_value *do_assign (const std::string& lhs, jit_value *rhs, bool print, + bool artificial = false); + + jit_value *visit (tree *tee) { return visit (*tee); } + + jit_value *visit (tree& tee); + + typedef std::list<jit_block *> block_list; + block_list breaks; + block_list continues; + + void finish_breaks (jit_block *dest, const block_list& lst); +}; + +// Convert from the low level Octave IR to LLVM +class +jit_convert_llvm : public jit_ir_walker +{ +public: + llvm::Function *convert_loop (llvm::Module *module, + const jit_block_list& blocks, + const std::list<jit_value *>& constants); + + jit_function convert_function (llvm::Module *module, + const jit_block_list& blocks, + const std::list<jit_value *>& constants, + octave_user_function& fcn, + const std::vector<jit_type *>& args); + + // arguments to the llvm::Function for loops + const std::vector<std::pair<std::string, bool> >& get_arguments(void) const + { return argument_vec; } + +#define JIT_METH(clname) \ + virtual void visit (jit_ ## clname&); + + JIT_VISIT_IR_CLASSES; + +#undef JIT_METH +private: + // name -> argument index (used for compiling functions) + std::map<std::string, int> argument_index; + + std::vector<std::pair<std::string, bool> > argument_vec; + + // name -> llvm argument (used for compiling loops) + std::map<std::string, llvm::Value *> arguments; + + bool converting_function; + + // only used if we are converting a function + jit_function creating; + + llvm::Function *function; + llvm::BasicBlock *prelude; + + void convert (const jit_block_list& blocks, + const std::list<jit_value *>& constants); + + void finish_phi (jit_phi *phi); + + void visit (jit_value *jvalue) + { + return visit (*jvalue); + } + + void visit (jit_value &jvalue) + { + jvalue.accept (*this); + } +}; + +// type inference and SSA construction on the low level Octave IR +class +jit_infer +{ +public: + typedef jit_convert::variable_map variable_map; + + jit_infer (jit_factory& afactory, jit_block_list& ablocks, + const variable_map& avmap); + + jit_block_list& get_blocks (void) const { return blocks; } + + jit_factory& get_factory (void) const { return factory; } + + void infer (void); +private: + jit_block_list& blocks; + jit_factory& factory; + const variable_map& vmap; + std::list<jit_instruction *> worklist; + + void append_users (jit_value *v); + + void append_users_term (jit_terminator *term); + + void construct_ssa (void); + + void do_construct_ssa (jit_block& block, size_t avisit_count); + + jit_block& entry_block (void) { return *blocks.front (); } + + jit_block& final_block (void) { return *blocks.back (); } + + void place_releases (void); + + void push_worklist (jit_instruction *instr); + + void remove_dead (); + + void release_dead_phi (jit_block& ablock); + + void release_temp (jit_block& ablock, std::set<jit_value *>& temp); + + void simplify_phi (void); + + void simplify_phi (jit_phi& phi); +}; + +class +tree_jit +{ +public: + ~tree_jit (void); + + static bool execute (tree_simple_for_command& cmd, + const octave_value& bounds); + + static bool execute (tree_while_command& cmd); + + static bool execute (octave_user_function& fcn, const octave_value_list& args, + octave_value_list& retval); + + llvm::ExecutionEngine *get_engine (void) const { return engine; } + + llvm::Module *get_module (void) const { return module; } + + void optimize (llvm::Function *fn); +private: + tree_jit (void); + + static tree_jit& instance (void); + + bool initialize (void); + + bool do_execute (tree_simple_for_command& cmd, const octave_value& bounds); + + bool do_execute (tree_while_command& cmd); + + bool do_execute (octave_user_function& fcn, const octave_value_list& args, + octave_value_list& retval); + + bool enabled (void); + + size_t trip_count (const octave_value& bounds) const; + + llvm::Module *module; +#if defined (LEGACY_PASSMANAGER) + llvm::legacy::PassManager *module_pass_manager; + llvm::legacy::FunctionPassManager *pass_manager; +#else + llvm::PassManager *module_pass_manager; + llvm::FunctionPassManager *pass_manager; +#endif + llvm::ExecutionEngine *engine; +}; + +class +jit_function_info +{ +public: + jit_function_info (tree_jit& tjit, octave_user_function& fcn, + const octave_value_list& ov_args); + + bool execute (const octave_value_list& ov_args, + octave_value_list& retval) const; + + bool match (const octave_value_list& ov_args) const; +private: + typedef octave_base_value *(*jited_function)(octave_base_value**); + + std::vector<jit_type *> argument_types; + jited_function function; +}; + +class +jit_info +{ +public: + // we use a pointer here so we don't have to include ov.h + typedef std::map<std::string, const octave_value *> vmap; + + jit_info (tree_jit& tjit, tree& tee); + + jit_info (tree_jit& tjit, tree& tee, const octave_value& for_bounds); + + ~jit_info (void); + + bool execute (const vmap& extra_vars = vmap ()) const; + + bool match (const vmap& extra_vars = vmap ()) const; +private: + typedef jit_convert::type_bound type_bound; + typedef jit_convert::type_bound_vector type_bound_vector; + typedef void (*jited_function)(octave_base_value**); + + void compile (tree_jit& tjit, tree& tee, jit_type *for_bounds = 0); + + octave_value find (const vmap& extra_vars, const std::string& vname) const; + + llvm::ExecutionEngine *engine; + jited_function function; + llvm::Function *llvm_function; + + std::vector<std::pair<std::string, bool> > arguments; + type_bound_vector bounds; +}; + +#endif +#endif