|
15016
|
1 /* |
|
|
2 |
|
|
3 Copyright (C) 2012 Max Brister <max@2bass.com> |
|
|
4 |
|
|
5 This file is part of Octave. |
|
|
6 |
|
|
7 Octave is free software; you can redistribute it and/or modify it |
|
|
8 under the terms of the GNU General Public License as published by the |
|
|
9 Free Software Foundation; either version 3 of the License, or (at your |
|
|
10 option) any later version. |
|
|
11 |
|
|
12 Octave is distributed in the hope that it will be useful, but WITHOUT |
|
|
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
|
15 for more details. |
|
|
16 |
|
|
17 You should have received a copy of the GNU General Public License |
|
|
18 along with Octave; see the file COPYING. If not, see |
|
|
19 <http://www.gnu.org/licenses/>. |
|
|
20 |
|
|
21 */ |
|
|
22 |
|
|
23 // defines required by llvm |
|
|
24 #define __STDC_LIMIT_MACROS |
|
|
25 #define __STDC_CONSTANT_MACROS |
|
|
26 |
|
|
27 #ifdef HAVE_CONFIG_H |
|
|
28 #include <config.h> |
|
|
29 #endif |
|
|
30 |
|
|
31 #ifdef HAVE_LLVM |
|
|
32 |
|
|
33 #include "jit-typeinfo.h" |
|
|
34 |
|
|
35 #include <llvm/Analysis/Verifier.h> |
|
|
36 #include <llvm/GlobalVariable.h> |
|
|
37 #include <llvm/ExecutionEngine/ExecutionEngine.h> |
|
|
38 #include <llvm/LLVMContext.h> |
|
|
39 #include <llvm/Function.h> |
|
|
40 #include <llvm/Instructions.h> |
|
|
41 #include <llvm/Intrinsics.h> |
|
|
42 #include <llvm/Support/IRBuilder.h> |
|
|
43 #include <llvm/Support/raw_os_ostream.h> |
|
|
44 |
|
|
45 #include "jit-ir.h" |
|
|
46 #include "ov.h" |
|
|
47 #include "ov-builtin.h" |
|
|
48 #include "ov-complex.h" |
|
|
49 #include "ov-scalar.h" |
|
|
50 #include "pager.h" |
|
|
51 |
|
|
52 static llvm::LLVMContext& context = llvm::getGlobalContext (); |
|
|
53 |
|
|
54 jit_typeinfo *jit_typeinfo::instance = 0; |
|
|
55 |
|
|
56 std::ostream& jit_print (std::ostream& os, jit_type *atype) |
|
|
57 { |
|
|
58 if (! atype) |
|
|
59 return os << "null"; |
|
|
60 return os << atype->name (); |
|
|
61 } |
|
|
62 |
|
|
63 // function that jit code calls |
|
|
64 extern "C" void |
|
|
65 octave_jit_print_any (const char *name, octave_base_value *obv) |
|
|
66 { |
|
|
67 obv->print_with_name (octave_stdout, name, true); |
|
|
68 } |
|
|
69 |
|
|
70 extern "C" void |
|
|
71 octave_jit_print_double (const char *name, double value) |
|
|
72 { |
|
|
73 // FIXME: We should avoid allocating a new octave_scalar each time |
|
|
74 octave_value ov (value); |
|
|
75 ov.print_with_name (octave_stdout, name); |
|
|
76 } |
|
|
77 |
|
|
78 extern "C" octave_base_value* |
|
|
79 octave_jit_binary_any_any (octave_value::binary_op op, octave_base_value *lhs, |
|
|
80 octave_base_value *rhs) |
|
|
81 { |
|
|
82 octave_value olhs (lhs, true); |
|
|
83 octave_value orhs (rhs, true); |
|
|
84 octave_value result = do_binary_op (op, olhs, orhs); |
|
|
85 octave_base_value *rep = result.internal_rep (); |
|
|
86 rep->grab (); |
|
|
87 return rep; |
|
|
88 } |
|
|
89 |
|
|
90 extern "C" octave_idx_type |
|
|
91 octave_jit_compute_nelem (double base, double limit, double inc) |
|
|
92 { |
|
|
93 Range rng = Range (base, limit, inc); |
|
|
94 return rng.nelem (); |
|
|
95 } |
|
|
96 |
|
|
97 extern "C" void |
|
|
98 octave_jit_release_any (octave_base_value *obv) |
|
|
99 { |
|
|
100 obv->release (); |
|
|
101 } |
|
|
102 |
|
|
103 extern "C" void |
|
|
104 octave_jit_release_matrix (jit_matrix *m) |
|
|
105 { |
|
|
106 delete m->array; |
|
|
107 } |
|
|
108 |
|
|
109 extern "C" octave_base_value * |
|
|
110 octave_jit_grab_any (octave_base_value *obv) |
|
|
111 { |
|
|
112 obv->grab (); |
|
|
113 return obv; |
|
|
114 } |
|
|
115 |
|
|
116 extern "C" void |
|
|
117 octave_jit_grab_matrix (jit_matrix *result, jit_matrix *m) |
|
|
118 { |
|
|
119 *result = *m->array; |
|
|
120 } |
|
|
121 |
|
|
122 extern "C" octave_base_value * |
|
|
123 octave_jit_cast_any_matrix (jit_matrix *m) |
|
|
124 { |
|
|
125 octave_value ret (*m->array); |
|
|
126 octave_base_value *rep = ret.internal_rep (); |
|
|
127 rep->grab (); |
|
|
128 delete m->array; |
|
|
129 |
|
|
130 return rep; |
|
|
131 } |
|
|
132 |
|
|
133 extern "C" void |
|
|
134 octave_jit_cast_matrix_any (jit_matrix *ret, octave_base_value *obv) |
|
|
135 { |
|
|
136 NDArray m = obv->array_value (); |
|
|
137 *ret = m; |
|
|
138 obv->release (); |
|
|
139 } |
|
|
140 |
|
|
141 extern "C" double |
|
|
142 octave_jit_cast_scalar_any (octave_base_value *obv) |
|
|
143 { |
|
|
144 double ret = obv->double_value (); |
|
|
145 obv->release (); |
|
|
146 return ret; |
|
|
147 } |
|
|
148 |
|
|
149 extern "C" octave_base_value * |
|
|
150 octave_jit_cast_any_scalar (double value) |
|
|
151 { |
|
|
152 return new octave_scalar (value); |
|
|
153 } |
|
|
154 |
|
|
155 extern "C" Complex |
|
|
156 octave_jit_cast_complex_any (octave_base_value *obv) |
|
|
157 { |
|
|
158 Complex ret = obv->complex_value (); |
|
|
159 obv->release (); |
|
|
160 return ret; |
|
|
161 } |
|
|
162 |
|
|
163 extern "C" octave_base_value * |
|
|
164 octave_jit_cast_any_complex (Complex c) |
|
|
165 { |
|
|
166 if (c.imag () == 0) |
|
|
167 return new octave_scalar (c.real ()); |
|
|
168 else |
|
|
169 return new octave_complex (c); |
|
|
170 } |
|
|
171 |
|
|
172 extern "C" void |
|
|
173 octave_jit_gripe_nan_to_logical_conversion (void) |
|
|
174 { |
|
|
175 try |
|
|
176 { |
|
|
177 gripe_nan_to_logical_conversion (); |
|
|
178 } |
|
|
179 catch (const octave_execution_exception&) |
|
|
180 { |
|
|
181 gripe_library_execution_error (); |
|
|
182 } |
|
|
183 } |
|
|
184 |
|
|
185 extern "C" void |
|
|
186 octave_jit_ginvalid_index (void) |
|
|
187 { |
|
|
188 try |
|
|
189 { |
|
|
190 gripe_invalid_index (); |
|
|
191 } |
|
|
192 catch (const octave_execution_exception&) |
|
|
193 { |
|
|
194 gripe_library_execution_error (); |
|
|
195 } |
|
|
196 } |
|
|
197 |
|
|
198 extern "C" void |
|
|
199 octave_jit_gindex_range (int nd, int dim, octave_idx_type iext, |
|
|
200 octave_idx_type ext) |
|
|
201 { |
|
|
202 try |
|
|
203 { |
|
|
204 gripe_index_out_of_range (nd, dim, iext, ext); |
|
|
205 } |
|
|
206 catch (const octave_execution_exception&) |
|
|
207 { |
|
|
208 gripe_library_execution_error (); |
|
|
209 } |
|
|
210 } |
|
|
211 |
|
|
212 extern "C" void |
|
|
213 octave_jit_paren_subsasgn_impl (jit_matrix *mat, octave_idx_type index, |
|
|
214 double value) |
|
|
215 { |
|
|
216 NDArray *array = mat->array; |
|
|
217 if (array->nelem () < index) |
|
|
218 array->resize1 (index); |
|
|
219 |
|
|
220 double *data = array->fortran_vec (); |
|
|
221 data[index - 1] = value; |
|
|
222 |
|
|
223 mat->update (); |
|
|
224 } |
|
|
225 |
|
|
226 extern "C" void |
|
|
227 octave_jit_paren_subsasgn_matrix_range (jit_matrix *result, jit_matrix *mat, |
|
|
228 jit_range *index, double value) |
|
|
229 { |
|
|
230 NDArray *array = mat->array; |
|
|
231 bool done = false; |
|
|
232 |
|
|
233 // optimize for the simple case (no resizing and no errors) |
|
|
234 if (*array->jit_ref_count () == 1 |
|
|
235 && index->all_elements_are_ints ()) |
|
|
236 { |
|
|
237 // this code is similar to idx_vector::fill, but we avoid allocating an |
|
|
238 // idx_vector and its associated rep |
|
|
239 octave_idx_type start = static_cast<octave_idx_type> (index->base) - 1; |
|
|
240 octave_idx_type step = static_cast<octave_idx_type> (index->inc); |
|
|
241 octave_idx_type nelem = index->nelem; |
|
|
242 octave_idx_type final = start + nelem * step; |
|
|
243 if (step < 0) |
|
|
244 { |
|
|
245 step = -step; |
|
|
246 std::swap (final, start); |
|
|
247 } |
|
|
248 |
|
|
249 if (start >= 0 && final < mat->slice_len) |
|
|
250 { |
|
|
251 done = true; |
|
|
252 |
|
|
253 double *data = array->jit_slice_data (); |
|
|
254 if (step == 1) |
|
|
255 std::fill (data + start, data + start + nelem, value); |
|
|
256 else |
|
|
257 { |
|
|
258 for (octave_idx_type i = start; i < final; i += step) |
|
|
259 data[i] = value; |
|
|
260 } |
|
|
261 } |
|
|
262 } |
|
|
263 |
|
|
264 if (! done) |
|
|
265 { |
|
|
266 idx_vector idx (*index); |
|
|
267 NDArray avalue (dim_vector (1, 1)); |
|
|
268 avalue.xelem (0) = value; |
|
|
269 array->assign (idx, avalue); |
|
|
270 } |
|
|
271 |
|
|
272 result->update (array); |
|
|
273 } |
|
|
274 |
|
|
275 extern "C" Complex |
|
|
276 octave_jit_complex_div (Complex lhs, Complex rhs) |
|
|
277 { |
|
|
278 // see src/OPERATORS/op-cs-cs.cc |
|
|
279 if (rhs == 0.0) |
|
|
280 gripe_divide_by_zero (); |
|
|
281 |
|
|
282 return lhs / rhs; |
|
|
283 } |
|
|
284 |
|
|
285 // FIXME: CP form src/xpow.cc |
|
|
286 static inline int |
|
|
287 xisint (double x) |
|
|
288 { |
|
|
289 return (D_NINT (x) == x |
|
|
290 && ((x >= 0 && x < INT_MAX) |
|
|
291 || (x <= 0 && x > INT_MIN))); |
|
|
292 } |
|
|
293 |
|
|
294 extern "C" Complex |
|
|
295 octave_jit_pow_scalar_scalar (double lhs, double rhs) |
|
|
296 { |
|
|
297 // FIXME: almost CP from src/xpow.cc |
|
|
298 if (lhs < 0.0 && ! xisint (rhs)) |
|
|
299 return std::pow (Complex (lhs), rhs); |
|
|
300 return std::pow (lhs, rhs); |
|
|
301 } |
|
|
302 |
|
|
303 extern "C" Complex |
|
|
304 octave_jit_pow_complex_complex (Complex lhs, Complex rhs) |
|
|
305 { |
|
|
306 if (lhs.imag () == 0 && rhs.imag () == 0) |
|
|
307 return octave_jit_pow_scalar_scalar (lhs.real (), rhs.real ()); |
|
|
308 return std::pow (lhs, rhs); |
|
|
309 } |
|
|
310 |
|
|
311 extern "C" Complex |
|
|
312 octave_jit_pow_complex_scalar (Complex lhs, double rhs) |
|
|
313 { |
|
|
314 if (lhs.imag () == 0) |
|
|
315 return octave_jit_pow_scalar_scalar (lhs.real (), rhs); |
|
|
316 return std::pow (lhs, rhs); |
|
|
317 } |
|
|
318 |
|
|
319 extern "C" Complex |
|
|
320 octave_jit_pow_scalar_complex (double lhs, Complex rhs) |
|
|
321 { |
|
|
322 if (rhs.imag () == 0) |
|
|
323 return octave_jit_pow_scalar_scalar (lhs, rhs.real ()); |
|
|
324 return std::pow (lhs, rhs); |
|
|
325 } |
|
|
326 |
|
|
327 extern "C" void |
|
|
328 octave_jit_print_matrix (jit_matrix *m) |
|
|
329 { |
|
|
330 std::cout << *m << std::endl; |
|
|
331 } |
|
|
332 |
|
|
333 static void |
|
|
334 gripe_bad_result (void) |
|
|
335 { |
|
|
336 error ("incorrect type information given to the JIT compiler"); |
|
|
337 } |
|
|
338 |
|
|
339 // FIXME: Add support for multiple outputs |
|
|
340 extern "C" octave_base_value * |
|
|
341 octave_jit_call (octave_builtin::fcn fn, size_t nargin, |
|
|
342 octave_base_value **argin, jit_type *result_type) |
|
|
343 { |
|
|
344 octave_value_list ovl (nargin); |
|
|
345 for (size_t i = 0; i < nargin; ++i) |
|
|
346 ovl.xelem (i) = octave_value (argin[i]); |
|
|
347 |
|
|
348 ovl = fn (ovl, 1); |
|
|
349 |
|
|
350 // These type checks are not strictly required, but I'm guessing that |
|
|
351 // incorrect types will be entered on occasion. This will be very difficult to |
|
|
352 // debug unless we do the sanity check here. |
|
|
353 if (result_type) |
|
|
354 { |
|
|
355 if (ovl.length () != 1) |
|
|
356 { |
|
|
357 gripe_bad_result (); |
|
|
358 return 0; |
|
|
359 } |
|
|
360 |
|
|
361 octave_value& result = ovl.xelem (0); |
|
|
362 jit_type *jtype = jit_typeinfo::join (jit_typeinfo::type_of (result), |
|
|
363 result_type); |
|
|
364 if (jtype != result_type) |
|
|
365 { |
|
|
366 gripe_bad_result (); |
|
|
367 return 0; |
|
|
368 } |
|
|
369 |
|
|
370 octave_base_value *ret = result.internal_rep (); |
|
|
371 ret->grab (); |
|
|
372 return ret; |
|
|
373 } |
|
|
374 |
|
|
375 if (! (ovl.length () == 0 |
|
|
376 || (ovl.length () == 1 && ovl.xelem (0).is_undefined ()))) |
|
|
377 gripe_bad_result (); |
|
|
378 |
|
|
379 return 0; |
|
|
380 } |
|
|
381 |
|
|
382 // -------------------- jit_range -------------------- |
|
|
383 bool |
|
|
384 jit_range::all_elements_are_ints () const |
|
|
385 { |
|
|
386 Range r (*this); |
|
|
387 return r.all_elements_are_ints (); |
|
|
388 } |
|
|
389 |
|
|
390 std::ostream& |
|
|
391 operator<< (std::ostream& os, const jit_range& rng) |
|
|
392 { |
|
|
393 return os << "Range[" << rng.base << ", " << rng.limit << ", " << rng.inc |
|
|
394 << ", " << rng.nelem << "]"; |
|
|
395 } |
|
|
396 |
|
|
397 // -------------------- jit_matrix -------------------- |
|
|
398 |
|
|
399 std::ostream& |
|
|
400 operator<< (std::ostream& os, const jit_matrix& mat) |
|
|
401 { |
|
|
402 return os << "Matrix[" << mat.ref_count << ", " << mat.slice_data << ", " |
|
|
403 << mat.slice_len << ", " << mat.dimensions << ", " |
|
|
404 << mat.array << "]"; |
|
|
405 } |
|
|
406 |
|
|
407 // -------------------- jit_type -------------------- |
|
|
408 jit_type::jit_type (const std::string& aname, jit_type *aparent, |
|
|
409 llvm::Type *allvm_type, int aid) : |
|
|
410 mname (aname), mparent (aparent), llvm_type (allvm_type), mid (aid), |
|
|
411 mdepth (aparent ? aparent->mdepth + 1 : 0) |
|
|
412 { |
|
|
413 std::memset (msret, 0, sizeof (msret)); |
|
|
414 std::memset (mpointer_arg, 0, sizeof (mpointer_arg)); |
|
|
415 std::memset (mpack, 0, sizeof (mpack)); |
|
|
416 std::memset (munpack, 0, sizeof (munpack)); |
|
|
417 |
|
|
418 for (size_t i = 0; i < jit_convention::length; ++i) |
|
|
419 mpacked_type[i] = llvm_type; |
|
|
420 } |
|
|
421 |
|
|
422 llvm::Type * |
|
|
423 jit_type::to_llvm_arg (void) const |
|
|
424 { |
|
|
425 return llvm_type ? llvm_type->getPointerTo () : 0; |
|
|
426 } |
|
|
427 |
|
|
428 // -------------------- jit_function -------------------- |
|
|
429 jit_function::jit_function () : module (0), llvm_function (0), mresult (0), |
|
|
430 call_conv (jit_convention::length), |
|
|
431 mcan_error (false) |
|
|
432 {} |
|
|
433 |
|
|
434 jit_function::jit_function (llvm::Module *amodule, |
|
|
435 jit_convention::type acall_conv, |
|
|
436 const llvm::Twine& aname, jit_type *aresult, |
|
|
437 const std::vector<jit_type *>& aargs) |
|
|
438 : module (amodule), mresult (aresult), args (aargs), call_conv (acall_conv), |
|
|
439 mcan_error (false) |
|
|
440 { |
|
|
441 llvm::SmallVector<llvm::Type *, 15> llvm_args; |
|
|
442 |
|
|
443 llvm::Type *rtype = llvm::Type::getVoidTy (context); |
|
|
444 if (mresult) |
|
|
445 { |
|
|
446 rtype = mresult->packed_type (call_conv); |
|
|
447 if (sret ()) |
|
|
448 { |
|
|
449 llvm_args.push_back (rtype->getPointerTo ()); |
|
|
450 rtype = llvm::Type::getVoidTy (context); |
|
|
451 } |
|
|
452 } |
|
|
453 |
|
|
454 for (std::vector<jit_type *>::const_iterator iter = args.begin (); |
|
|
455 iter != args.end (); ++iter) |
|
|
456 { |
|
|
457 jit_type *ty = *iter; |
|
|
458 assert (ty); |
|
|
459 llvm::Type *argty = ty->packed_type (call_conv); |
|
|
460 if (ty->pointer_arg (call_conv)) |
|
|
461 argty = argty->getPointerTo (); |
|
|
462 |
|
|
463 llvm_args.push_back (argty); |
|
|
464 } |
|
|
465 |
|
|
466 // we mark all functinos as external linkage because this prevents llvm |
|
|
467 // from getting rid of always inline functions |
|
|
468 llvm::FunctionType *ft = llvm::FunctionType::get (rtype, llvm_args, false); |
|
|
469 llvm_function = llvm::Function::Create (ft, llvm::Function::ExternalLinkage, |
|
|
470 aname, module); |
|
|
471 if (call_conv == jit_convention::internal) |
|
|
472 llvm_function->addFnAttr (llvm::Attribute::AlwaysInline); |
|
|
473 } |
|
|
474 |
|
|
475 jit_function::jit_function (const jit_function& fn, jit_type *aresult, |
|
|
476 const std::vector<jit_type *>& aargs) |
|
|
477 : module (fn.module), llvm_function (fn.llvm_function), mresult (aresult), |
|
|
478 args (aargs), call_conv (fn.call_conv), mcan_error (fn.mcan_error) |
|
|
479 { |
|
|
480 } |
|
|
481 |
|
|
482 jit_function::jit_function (const jit_function& fn) |
|
|
483 : module (fn.module), llvm_function (fn.llvm_function), mresult (fn.mresult), |
|
|
484 args (fn.args), call_conv (fn.call_conv), mcan_error (fn.mcan_error) |
|
|
485 {} |
|
|
486 |
|
|
487 std::string |
|
|
488 jit_function::name (void) const |
|
|
489 { |
|
|
490 return llvm_function->getName (); |
|
|
491 } |
|
|
492 |
|
|
493 llvm::BasicBlock * |
|
|
494 jit_function::new_block (const std::string& aname, |
|
|
495 llvm::BasicBlock *insert_before) |
|
|
496 { |
|
|
497 return llvm::BasicBlock::Create (context, aname, llvm_function, |
|
|
498 insert_before); |
|
|
499 } |
|
|
500 |
|
|
501 llvm::Value * |
|
|
502 jit_function::call (llvm::IRBuilderD& builder, |
|
|
503 const std::vector<jit_value *>& in_args) const |
|
|
504 { |
|
|
505 assert (in_args.size () == args.size ()); |
|
|
506 |
|
|
507 std::vector<llvm::Value *> llvm_args (args.size ()); |
|
|
508 for (size_t i = 0; i < in_args.size (); ++i) |
|
|
509 llvm_args[i] = in_args[i]->to_llvm (); |
|
|
510 |
|
|
511 return call (builder, llvm_args); |
|
|
512 } |
|
|
513 |
|
|
514 llvm::Value * |
|
|
515 jit_function::call (llvm::IRBuilderD& builder, |
|
|
516 const std::vector<llvm::Value *>& in_args) const |
|
|
517 { |
|
|
518 assert (valid ()); |
|
|
519 assert (in_args.size () == args.size ()); |
|
|
520 llvm::Function *stacksave |
|
|
521 = llvm::Intrinsic::getDeclaration (module, llvm::Intrinsic::stacksave); |
|
|
522 llvm::SmallVector<llvm::Value *, 10> llvm_args; |
|
|
523 llvm_args.reserve (in_args.size () + sret ()); |
|
|
524 |
|
|
525 llvm::Value *sret_mem = 0; |
|
|
526 llvm::Value *saved_stack = 0; |
|
|
527 if (sret ()) |
|
|
528 { |
|
|
529 saved_stack = builder.CreateCall (stacksave); |
|
|
530 sret_mem = builder.CreateAlloca (mresult->packed_type (call_conv)); |
|
|
531 llvm_args.push_back (sret_mem); |
|
|
532 } |
|
|
533 |
|
|
534 for (size_t i = 0; i < in_args.size (); ++i) |
|
|
535 { |
|
|
536 llvm::Value *arg = in_args[i]; |
|
|
537 jit_type::convert_fn convert = args[i]->pack (call_conv); |
|
|
538 if (convert) |
|
|
539 arg = convert (builder, arg); |
|
|
540 |
|
|
541 if (args[i]->pointer_arg (call_conv)) |
|
|
542 { |
|
|
543 if (! saved_stack) |
|
|
544 saved_stack = builder.CreateCall (stacksave); |
|
|
545 |
|
|
546 arg = builder.CreateAlloca (args[i]->to_llvm ()); |
|
|
547 builder.CreateStore (in_args[i], arg); |
|
|
548 } |
|
|
549 |
|
|
550 llvm_args.push_back (arg); |
|
|
551 } |
|
|
552 |
|
|
553 llvm::Value *ret = builder.CreateCall (llvm_function, llvm_args); |
|
|
554 if (sret_mem) |
|
|
555 ret = builder.CreateLoad (sret_mem); |
|
|
556 |
|
|
557 if (mresult) |
|
|
558 { |
|
|
559 jit_type::convert_fn unpack = mresult->unpack (call_conv); |
|
|
560 if (unpack) |
|
|
561 ret = unpack (builder, ret); |
|
|
562 } |
|
|
563 |
|
|
564 if (saved_stack) |
|
|
565 { |
|
|
566 llvm::Function *stackrestore |
|
|
567 = llvm::Intrinsic::getDeclaration (module, |
|
|
568 llvm::Intrinsic::stackrestore); |
|
|
569 builder.CreateCall (stackrestore, saved_stack); |
|
|
570 } |
|
|
571 |
|
|
572 return ret; |
|
|
573 } |
|
|
574 |
|
|
575 llvm::Value * |
|
|
576 jit_function::argument (llvm::IRBuilderD& builder, size_t idx) const |
|
|
577 { |
|
|
578 assert (idx < args.size ()); |
|
|
579 |
|
|
580 // FIXME: We should be treating arguments like a list, not a vector. Shouldn't |
|
|
581 // matter much for now, as the number of arguments shouldn't be much bigger |
|
|
582 // than 4 |
|
|
583 llvm::Function::arg_iterator iter = llvm_function->arg_begin (); |
|
|
584 if (sret ()) |
|
|
585 ++iter; |
|
|
586 |
|
|
587 for (size_t i = 0; i < idx; ++i, ++iter); |
|
|
588 |
|
|
589 if (args[idx]->pointer_arg (call_conv)) |
|
|
590 return builder.CreateLoad (iter); |
|
|
591 |
|
|
592 return iter; |
|
|
593 } |
|
|
594 |
|
|
595 void |
|
|
596 jit_function::do_return (llvm::IRBuilderD& builder, llvm::Value *rval) |
|
|
597 { |
|
|
598 assert (! rval == ! mresult); |
|
|
599 |
|
|
600 if (rval) |
|
|
601 { |
|
|
602 jit_type::convert_fn convert = mresult->pack (call_conv); |
|
|
603 if (convert) |
|
|
604 rval = convert (builder, rval); |
|
|
605 |
|
|
606 if (sret ()) |
|
|
607 builder.CreateStore (rval, llvm_function->arg_begin ()); |
|
|
608 else |
|
|
609 builder.CreateRet (rval); |
|
|
610 } |
|
|
611 else |
|
|
612 builder.CreateRetVoid (); |
|
|
613 |
|
|
614 llvm::verifyFunction (*llvm_function); |
|
|
615 } |
|
|
616 |
|
|
617 std::ostream& |
|
|
618 operator<< (std::ostream& os, const jit_function& fn) |
|
|
619 { |
|
|
620 llvm::Function *lfn = fn.to_llvm (); |
|
|
621 os << "jit_function: cc=" << fn.call_conv; |
|
|
622 llvm::raw_os_ostream llvm_out (os); |
|
|
623 lfn->print (llvm_out); |
|
|
624 llvm_out.flush (); |
|
|
625 return os; |
|
|
626 } |
|
|
627 |
|
|
628 // -------------------- jit_operation -------------------- |
|
|
629 void |
|
|
630 jit_operation::add_overload (const jit_function& func, |
|
|
631 const std::vector<jit_type*>& args) |
|
|
632 { |
|
|
633 if (args.size () >= overloads.size ()) |
|
|
634 overloads.resize (args.size () + 1); |
|
|
635 |
|
|
636 Array<jit_function>& over = overloads[args.size ()]; |
|
|
637 dim_vector dv (over.dims ()); |
|
|
638 Array<octave_idx_type> idx = to_idx (args); |
|
|
639 bool must_resize = false; |
|
|
640 |
|
|
641 if (dv.length () != idx.numel ()) |
|
|
642 { |
|
|
643 dv.resize (idx.numel ()); |
|
|
644 must_resize = true; |
|
|
645 } |
|
|
646 |
|
|
647 for (octave_idx_type i = 0; i < dv.length (); ++i) |
|
|
648 if (dv(i) <= idx(i)) |
|
|
649 { |
|
|
650 must_resize = true; |
|
|
651 dv(i) = idx(i) + 1; |
|
|
652 } |
|
|
653 |
|
|
654 if (must_resize) |
|
|
655 over.resize (dv); |
|
|
656 |
|
|
657 over(idx) = func; |
|
|
658 } |
|
|
659 |
|
|
660 const jit_function& |
|
|
661 jit_operation::overload (const std::vector<jit_type*>& types) const |
|
|
662 { |
|
|
663 // FIXME: We should search for the next best overload on failure |
|
|
664 static jit_function null_overload; |
|
|
665 if (types.size () >= overloads.size ()) |
|
|
666 return null_overload; |
|
|
667 |
|
|
668 for (size_t i =0; i < types.size (); ++i) |
|
|
669 if (! types[i]) |
|
|
670 return null_overload; |
|
|
671 |
|
|
672 const Array<jit_function>& over = overloads[types.size ()]; |
|
|
673 dim_vector dv (over.dims ()); |
|
|
674 Array<octave_idx_type> idx = to_idx (types); |
|
|
675 for (octave_idx_type i = 0; i < dv.length (); ++i) |
|
|
676 if (idx(i) >= dv(i)) |
|
|
677 return null_overload; |
|
|
678 |
|
|
679 return over(idx); |
|
|
680 } |
|
|
681 |
|
|
682 Array<octave_idx_type> |
|
|
683 jit_operation::to_idx (const std::vector<jit_type*>& types) const |
|
|
684 { |
|
|
685 octave_idx_type numel = types.size (); |
|
|
686 if (numel == 1) |
|
|
687 numel = 2; |
|
|
688 |
|
|
689 Array<octave_idx_type> idx (dim_vector (1, numel)); |
|
|
690 for (octave_idx_type i = 0; i < static_cast<octave_idx_type> (types.size ()); |
|
|
691 ++i) |
|
|
692 idx(i) = types[i]->type_id (); |
|
|
693 |
|
|
694 if (types.size () == 1) |
|
|
695 { |
|
|
696 idx(1) = idx(0); |
|
|
697 idx(0) = 0; |
|
|
698 } |
|
|
699 |
|
|
700 return idx; |
|
|
701 } |
|
|
702 |
|
|
703 // -------------------- jit_typeinfo -------------------- |
|
|
704 void |
|
|
705 jit_typeinfo::initialize (llvm::Module *m, llvm::ExecutionEngine *e) |
|
|
706 { |
|
|
707 new jit_typeinfo (m, e); |
|
|
708 } |
|
|
709 |
|
|
710 jit_typeinfo::jit_typeinfo (llvm::Module *m, llvm::ExecutionEngine *e) |
|
|
711 : module (m), engine (e), next_id (0), |
|
|
712 builder (*new llvm::IRBuilderD (context)) |
|
|
713 { |
|
|
714 instance = this; |
|
|
715 |
|
|
716 // FIXME: We should be registering types like in octave_value_typeinfo |
|
|
717 llvm::Type *any_t = llvm::StructType::create (context, "octave_base_value"); |
|
|
718 any_t = any_t->getPointerTo (); |
|
|
719 |
|
|
720 llvm::Type *scalar_t = llvm::Type::getDoubleTy (context); |
|
|
721 llvm::Type *bool_t = llvm::Type::getInt1Ty (context); |
|
|
722 llvm::Type *string_t = llvm::Type::getInt8Ty (context); |
|
|
723 string_t = string_t->getPointerTo (); |
|
|
724 llvm::Type *index_t = llvm::Type::getIntNTy (context, |
|
|
725 sizeof(octave_idx_type) * 8); |
|
|
726 |
|
|
727 llvm::StructType *range_t = llvm::StructType::create (context, "range"); |
|
|
728 std::vector<llvm::Type *> range_contents (4, scalar_t); |
|
|
729 range_contents[3] = index_t; |
|
|
730 range_t->setBody (range_contents); |
|
|
731 |
|
|
732 llvm::Type *refcount_t = llvm::Type::getIntNTy (context, sizeof(int) * 8); |
|
|
733 |
|
|
734 llvm::StructType *matrix_t = llvm::StructType::create (context, "matrix"); |
|
|
735 llvm::Type *matrix_contents[5]; |
|
|
736 matrix_contents[0] = refcount_t->getPointerTo (); |
|
|
737 matrix_contents[1] = scalar_t->getPointerTo (); |
|
|
738 matrix_contents[2] = index_t; |
|
|
739 matrix_contents[3] = index_t->getPointerTo (); |
|
|
740 matrix_contents[4] = string_t; |
|
|
741 matrix_t->setBody (llvm::makeArrayRef (matrix_contents, 5)); |
|
|
742 |
|
|
743 llvm::Type *complex_t = llvm::VectorType::get (scalar_t, 2); |
|
|
744 |
|
|
745 // complex_ret is what is passed to C functions in order to get calling |
|
|
746 // convention right |
|
|
747 complex_ret = llvm::StructType::create (context, "complex_ret"); |
|
|
748 llvm::Type *complex_ret_contents[] = {scalar_t, scalar_t}; |
|
|
749 complex_ret->setBody (complex_ret_contents); |
|
|
750 |
|
|
751 // create types |
|
|
752 any = new_type ("any", 0, any_t); |
|
|
753 matrix = new_type ("matrix", any, matrix_t); |
|
|
754 complex = new_type ("complex", any, complex_t); |
|
|
755 scalar = new_type ("scalar", complex, scalar_t); |
|
|
756 range = new_type ("range", any, range_t); |
|
|
757 string = new_type ("string", any, string_t); |
|
|
758 boolean = new_type ("bool", any, bool_t); |
|
|
759 index = new_type ("index", any, index_t); |
|
|
760 |
|
|
761 create_int (8); |
|
|
762 create_int (16); |
|
|
763 create_int (32); |
|
|
764 create_int (64); |
|
|
765 |
|
|
766 casts.resize (next_id + 1); |
|
|
767 identities.resize (next_id + 1); |
|
|
768 |
|
|
769 // specify calling conventions |
|
|
770 // FIXME: We should detect architecture and do something sane based on that |
|
|
771 // here we assume x86 or x86_64 |
|
|
772 matrix->mark_sret (); |
|
|
773 matrix->mark_pointer_arg (); |
|
|
774 |
|
|
775 range->mark_sret (); |
|
|
776 range->mark_pointer_arg (); |
|
|
777 |
|
|
778 complex->set_pack (jit_convention::external, &jit_typeinfo::pack_complex); |
|
|
779 complex->set_unpack (jit_convention::external, &jit_typeinfo::unpack_complex); |
|
|
780 complex->set_packed_type (jit_convention::external, complex_ret); |
|
|
781 |
|
|
782 if (sizeof (void *) == 4) |
|
|
783 complex->mark_sret (); |
|
|
784 |
|
|
785 // bind global variables |
|
|
786 lerror_state = new llvm::GlobalVariable (*module, bool_t, false, |
|
|
787 llvm::GlobalValue::ExternalLinkage, |
|
|
788 0, "error_state"); |
|
|
789 engine->addGlobalMapping (lerror_state, |
|
|
790 reinterpret_cast<void *> (&error_state)); |
|
|
791 |
|
|
792 // any with anything is an any op |
|
|
793 jit_function fn; |
|
|
794 jit_type *binary_op_type = intN (sizeof (octave_value::binary_op) * 8); |
|
|
795 llvm::Type *llvm_bo_type = binary_op_type->to_llvm (); |
|
|
796 jit_function any_binary = create_function (jit_convention::external, |
|
|
797 "octave_jit_binary_any_any", |
|
|
798 any, binary_op_type, any, any); |
|
|
799 any_binary.mark_can_error (); |
|
|
800 binary_ops.resize (octave_value::num_binary_ops); |
|
|
801 for (size_t i = 0; i < octave_value::num_binary_ops; ++i) |
|
|
802 { |
|
|
803 octave_value::binary_op op = static_cast<octave_value::binary_op> (i); |
|
|
804 std::string op_name = octave_value::binary_op_as_string (op); |
|
|
805 binary_ops[i].stash_name ("binary" + op_name); |
|
|
806 } |
|
|
807 |
|
|
808 for (int op = 0; op < octave_value::num_binary_ops; ++op) |
|
|
809 { |
|
|
810 llvm::Twine fn_name ("octave_jit_binary_any_any_"); |
|
|
811 fn_name = fn_name + llvm::Twine (op); |
|
|
812 |
|
|
813 fn = create_function (jit_convention::internal, fn_name, any, any, any); |
|
|
814 fn.mark_can_error (); |
|
|
815 llvm::BasicBlock *block = fn.new_block (); |
|
|
816 builder.SetInsertPoint (block); |
|
|
817 llvm::APInt op_int(sizeof (octave_value::binary_op) * 8, op, |
|
|
818 std::numeric_limits<octave_value::binary_op>::is_signed); |
|
|
819 llvm::Value *op_as_llvm = llvm::ConstantInt::get (llvm_bo_type, op_int); |
|
|
820 llvm::Value *ret = any_binary.call (builder, op_as_llvm, |
|
|
821 fn.argument (builder, 0), |
|
|
822 fn.argument (builder, 1)); |
|
|
823 fn.do_return (builder, ret); |
|
|
824 binary_ops[op].add_overload (fn); |
|
|
825 } |
|
|
826 |
|
|
827 // grab any |
|
|
828 fn = create_function (jit_convention::external, "octave_jit_grab_any", any, |
|
|
829 any); |
|
|
830 grab_fn.add_overload (fn); |
|
|
831 grab_fn.stash_name ("grab"); |
|
|
832 |
|
|
833 // grab matrix |
|
|
834 fn = create_function (jit_convention::external, "octave_jit_grab_matrix", |
|
|
835 matrix, matrix); |
|
|
836 grab_fn.add_overload (fn); |
|
|
837 |
|
|
838 // release any |
|
|
839 fn = create_function (jit_convention::external, "octave_jit_release_any", 0, |
|
|
840 any); |
|
|
841 release_fn.add_overload (fn); |
|
|
842 release_fn.stash_name ("release"); |
|
|
843 |
|
|
844 // release matrix |
|
|
845 fn = create_function (jit_convention::external, "octave_jit_release_matrix", |
|
|
846 0, matrix); |
|
|
847 release_fn.add_overload (fn); |
|
|
848 |
|
|
849 // release scalar |
|
|
850 fn = create_identity (scalar); |
|
|
851 release_fn.add_overload (fn); |
|
|
852 |
|
|
853 // release complex |
|
|
854 fn = create_identity (complex); |
|
|
855 release_fn.add_overload (fn); |
|
|
856 |
|
|
857 // release index |
|
|
858 fn = create_identity (index); |
|
|
859 release_fn.add_overload (fn); |
|
|
860 |
|
|
861 // now for binary scalar operations |
|
|
862 // FIXME: Finish all operations |
|
|
863 add_binary_op (scalar, octave_value::op_add, llvm::Instruction::FAdd); |
|
|
864 add_binary_op (scalar, octave_value::op_sub, llvm::Instruction::FSub); |
|
|
865 add_binary_op (scalar, octave_value::op_mul, llvm::Instruction::FMul); |
|
|
866 add_binary_op (scalar, octave_value::op_el_mul, llvm::Instruction::FMul); |
|
|
867 |
|
|
868 add_binary_fcmp (scalar, octave_value::op_lt, llvm::CmpInst::FCMP_ULT); |
|
|
869 add_binary_fcmp (scalar, octave_value::op_le, llvm::CmpInst::FCMP_ULE); |
|
|
870 add_binary_fcmp (scalar, octave_value::op_eq, llvm::CmpInst::FCMP_UEQ); |
|
|
871 add_binary_fcmp (scalar, octave_value::op_ge, llvm::CmpInst::FCMP_UGE); |
|
|
872 add_binary_fcmp (scalar, octave_value::op_gt, llvm::CmpInst::FCMP_UGT); |
|
|
873 add_binary_fcmp (scalar, octave_value::op_ne, llvm::CmpInst::FCMP_UNE); |
|
|
874 |
|
|
875 jit_function gripe_div0 = create_function (jit_convention::external, |
|
|
876 "gripe_divide_by_zero", 0); |
|
|
877 gripe_div0.mark_can_error (); |
|
|
878 |
|
|
879 // divide is annoying because it might error |
|
|
880 fn = create_function (jit_convention::internal, |
|
|
881 "octave_jit_div_scalar_scalar", scalar, scalar, scalar); |
|
|
882 fn.mark_can_error (); |
|
|
883 |
|
|
884 llvm::BasicBlock *body = fn.new_block (); |
|
|
885 builder.SetInsertPoint (body); |
|
|
886 { |
|
|
887 llvm::BasicBlock *warn_block = fn.new_block ("warn"); |
|
|
888 llvm::BasicBlock *normal_block = fn.new_block ("normal"); |
|
|
889 |
|
|
890 llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); |
|
|
891 llvm::Value *check = builder.CreateFCmpUEQ (zero, fn.argument (builder, 0)); |
|
|
892 builder.CreateCondBr (check, warn_block, normal_block); |
|
|
893 |
|
|
894 builder.SetInsertPoint (warn_block); |
|
|
895 gripe_div0.call (builder); |
|
|
896 builder.CreateBr (normal_block); |
|
|
897 |
|
|
898 builder.SetInsertPoint (normal_block); |
|
|
899 llvm::Value *ret = builder.CreateFDiv (fn.argument (builder, 0), |
|
|
900 fn.argument (builder, 1)); |
|
|
901 fn.do_return (builder, ret); |
|
|
902 } |
|
|
903 binary_ops[octave_value::op_div].add_overload (fn); |
|
|
904 binary_ops[octave_value::op_el_div].add_overload (fn); |
|
|
905 |
|
|
906 // ldiv is the same as div with the operators reversed |
|
|
907 fn = mirror_binary (fn); |
|
|
908 binary_ops[octave_value::op_ldiv].add_overload (fn); |
|
|
909 binary_ops[octave_value::op_el_ldiv].add_overload (fn); |
|
|
910 |
|
|
911 // In general, the result of scalar ^ scalar is a complex number. We might be |
|
|
912 // able to improve on this if we keep track of the range of values varaibles |
|
|
913 // can take on. |
|
|
914 fn = create_function (jit_convention::external, |
|
|
915 "octave_jit_pow_scalar_scalar", complex, scalar, |
|
|
916 scalar); |
|
|
917 binary_ops[octave_value::op_pow].add_overload (fn); |
|
|
918 binary_ops[octave_value::op_el_pow].add_overload (fn); |
|
|
919 |
|
|
920 // now for binary complex operations |
|
|
921 add_binary_op (complex, octave_value::op_add, llvm::Instruction::FAdd); |
|
|
922 add_binary_op (complex, octave_value::op_sub, llvm::Instruction::FSub); |
|
|
923 |
|
|
924 fn = create_function (jit_convention::internal, |
|
|
925 "octave_jit_*_complex_complex", complex, complex, |
|
|
926 complex); |
|
|
927 body = fn.new_block (); |
|
|
928 builder.SetInsertPoint (body); |
|
|
929 { |
|
|
930 // (x0*x1 - y0*y1, x0*y1 + y0*x1) = (x0,y0) * (x1,y1) |
|
|
931 // We compute this in one vectorized multiplication, a subtraction, and an |
|
|
932 // addition. |
|
|
933 llvm::Value *lhs = fn.argument (builder, 0); |
|
|
934 llvm::Value *rhs = fn.argument (builder, 1); |
|
|
935 |
|
|
936 // FIXME: We need a better way of doing this, working with llvm's IR |
|
|
937 // directly is sort of a pain. |
|
|
938 llvm::Value *zero = builder.getInt32 (0); |
|
|
939 llvm::Value *one = builder.getInt32 (1); |
|
|
940 llvm::Value *two = builder.getInt32 (2); |
|
|
941 llvm::Value *three = builder.getInt32 (3); |
|
|
942 |
|
|
943 llvm::Type *vec4 = llvm::VectorType::get (scalar_t, 4); |
|
|
944 llvm::Value *mlhs = llvm::UndefValue::get (vec4); |
|
|
945 llvm::Value *mrhs = mlhs; |
|
|
946 |
|
|
947 llvm::Value *temp = complex_real (lhs); |
|
|
948 mlhs = builder.CreateInsertElement (mlhs, temp, zero); |
|
|
949 mlhs = builder.CreateInsertElement (mlhs, temp, two); |
|
|
950 temp = complex_imag (lhs); |
|
|
951 mlhs = builder.CreateInsertElement (mlhs, temp, one); |
|
|
952 mlhs = builder.CreateInsertElement (mlhs, temp, three); |
|
|
953 |
|
|
954 temp = complex_real (rhs); |
|
|
955 mrhs = builder.CreateInsertElement (mrhs, temp, zero); |
|
|
956 mrhs = builder.CreateInsertElement (mrhs, temp, three); |
|
|
957 temp = complex_imag (rhs); |
|
|
958 mrhs = builder.CreateInsertElement (mrhs, temp, one); |
|
|
959 mrhs = builder.CreateInsertElement (mrhs, temp, two); |
|
|
960 |
|
|
961 llvm::Value *mres = builder.CreateFMul (mlhs, mrhs); |
|
|
962 llvm::Value *tlhs = builder.CreateExtractElement (mres, zero); |
|
|
963 llvm::Value *trhs = builder.CreateExtractElement (mres, one); |
|
|
964 llvm::Value *ret_real = builder.CreateFSub (tlhs, trhs); |
|
|
965 |
|
|
966 tlhs = builder.CreateExtractElement (mres, two); |
|
|
967 trhs = builder.CreateExtractElement (mres, three); |
|
|
968 llvm::Value *ret_imag = builder.CreateFAdd (tlhs, trhs); |
|
|
969 fn.do_return (builder, complex_new (ret_real, ret_imag)); |
|
|
970 } |
|
|
971 |
|
|
972 binary_ops[octave_value::op_mul].add_overload (fn); |
|
|
973 binary_ops[octave_value::op_el_mul].add_overload (fn); |
|
|
974 |
|
|
975 jit_function complex_div = create_function (jit_convention::external, |
|
|
976 "octave_jit_complex_div", |
|
|
977 complex, complex, complex); |
|
|
978 complex_div.mark_can_error (); |
|
|
979 binary_ops[octave_value::op_div].add_overload (fn); |
|
|
980 binary_ops[octave_value::op_ldiv].add_overload (fn); |
|
|
981 |
|
|
982 fn = mirror_binary (complex_div); |
|
|
983 binary_ops[octave_value::op_ldiv].add_overload (fn); |
|
|
984 binary_ops[octave_value::op_el_ldiv].add_overload (fn); |
|
|
985 |
|
|
986 fn = create_function (jit_convention::external, |
|
|
987 "octave_jit_pow_complex_complex", complex, complex, |
|
|
988 complex); |
|
|
989 binary_ops[octave_value::op_pow].add_overload (fn); |
|
|
990 binary_ops[octave_value::op_el_pow].add_overload (fn); |
|
|
991 |
|
|
992 fn = create_function (jit_convention::internal, |
|
|
993 "octave_jit_*_scalar_complex", complex, scalar, |
|
|
994 complex); |
|
|
995 jit_function mul_scalar_complex = fn; |
|
|
996 body = fn.new_block (); |
|
|
997 builder.SetInsertPoint (body); |
|
|
998 { |
|
|
999 llvm::Value *lhs = fn.argument (builder, 0); |
|
|
1000 llvm::Value *tlhs = complex_new (lhs, lhs); |
|
|
1001 llvm::Value *rhs = fn.argument (builder, 1); |
|
|
1002 fn.do_return (builder, builder.CreateFMul (tlhs, rhs)); |
|
|
1003 } |
|
|
1004 binary_ops[octave_value::op_mul].add_overload (fn); |
|
|
1005 binary_ops[octave_value::op_el_mul].add_overload (fn); |
|
|
1006 |
|
|
1007 |
|
|
1008 fn = mirror_binary (mul_scalar_complex); |
|
|
1009 binary_ops[octave_value::op_mul].add_overload (fn); |
|
|
1010 binary_ops[octave_value::op_el_mul].add_overload (fn); |
|
|
1011 |
|
|
1012 fn = create_function (jit_convention::internal, "octave_jit_+_scalar_complex", |
|
|
1013 complex, scalar, complex); |
|
|
1014 body = fn.new_block (); |
|
|
1015 builder.SetInsertPoint (body); |
|
|
1016 { |
|
|
1017 llvm::Value *lhs = fn.argument (builder, 0); |
|
|
1018 llvm::Value *rhs = fn.argument (builder, 1); |
|
|
1019 llvm::Value *real = builder.CreateFAdd (lhs, complex_real (rhs)); |
|
|
1020 fn.do_return (builder, complex_real (rhs, real)); |
|
|
1021 } |
|
|
1022 binary_ops[octave_value::op_add].add_overload (fn); |
|
|
1023 |
|
|
1024 fn = mirror_binary (fn); |
|
|
1025 binary_ops[octave_value::op_add].add_overload (fn); |
|
|
1026 |
|
|
1027 fn = create_function (jit_convention::internal, "octave_jit_-_complex_scalar", |
|
|
1028 complex, complex, scalar); |
|
|
1029 body = fn.new_block (); |
|
|
1030 builder.SetInsertPoint (body); |
|
|
1031 { |
|
|
1032 llvm::Value *lhs = fn.argument (builder, 0); |
|
|
1033 llvm::Value *rhs = fn.argument (builder, 1); |
|
|
1034 llvm::Value *real = builder.CreateFSub (complex_real (lhs), rhs); |
|
|
1035 fn.do_return (builder, complex_real (lhs, real)); |
|
|
1036 } |
|
|
1037 binary_ops[octave_value::op_sub].add_overload (fn); |
|
|
1038 |
|
|
1039 fn = create_function (jit_convention::internal, "octave_jit_-_scalar_complex", |
|
|
1040 complex, scalar, complex); |
|
|
1041 body = fn.new_block (); |
|
|
1042 builder.SetInsertPoint (body); |
|
|
1043 { |
|
|
1044 llvm::Value *lhs = fn.argument (builder, 0); |
|
|
1045 llvm::Value *rhs = fn.argument (builder, 1); |
|
|
1046 llvm::Value *real = builder.CreateFSub (lhs, complex_real (rhs)); |
|
|
1047 fn.do_return (builder, complex_real (rhs, real)); |
|
|
1048 } |
|
|
1049 binary_ops[octave_value::op_sub].add_overload (fn); |
|
|
1050 |
|
|
1051 fn = create_function (jit_convention::external, |
|
|
1052 "octave_jit_pow_scalar_complex", complex, scalar, |
|
|
1053 complex); |
|
|
1054 binary_ops[octave_value::op_pow].add_overload (fn); |
|
|
1055 binary_ops[octave_value::op_el_pow].add_overload (fn); |
|
|
1056 |
|
|
1057 fn = create_function (jit_convention::external, |
|
|
1058 "octave_jit_pow_complex_scalar", complex, complex, |
|
|
1059 scalar); |
|
|
1060 binary_ops[octave_value::op_pow].add_overload (fn); |
|
|
1061 binary_ops[octave_value::op_el_pow].add_overload (fn); |
|
|
1062 |
|
|
1063 // now for binary index operators |
|
|
1064 add_binary_op (index, octave_value::op_add, llvm::Instruction::Add); |
|
|
1065 |
|
|
1066 // and binary bool operators |
|
|
1067 add_binary_op (boolean, octave_value::op_el_or, llvm::Instruction::Or); |
|
|
1068 add_binary_op (boolean, octave_value::op_el_and, llvm::Instruction::And); |
|
|
1069 |
|
|
1070 // now for printing functions |
|
|
1071 print_fn.stash_name ("print"); |
|
|
1072 add_print (any); |
|
|
1073 add_print (scalar); |
|
|
1074 |
|
|
1075 // initialize for loop |
|
|
1076 for_init_fn.stash_name ("for_init"); |
|
|
1077 |
|
|
1078 fn = create_function (jit_convention::internal, "octave_jit_for_range_init", |
|
|
1079 index, range); |
|
|
1080 body = fn.new_block (); |
|
|
1081 builder.SetInsertPoint (body); |
|
|
1082 { |
|
|
1083 llvm::Value *zero = llvm::ConstantInt::get (index_t, 0); |
|
|
1084 fn.do_return (builder, zero); |
|
|
1085 } |
|
|
1086 for_init_fn.add_overload (fn); |
|
|
1087 |
|
|
1088 // bounds check for for loop |
|
|
1089 for_check_fn.stash_name ("for_check"); |
|
|
1090 |
|
|
1091 fn = create_function (jit_convention::internal, "octave_jit_for_range_check", |
|
|
1092 boolean, range, index); |
|
|
1093 body = fn.new_block (); |
|
|
1094 builder.SetInsertPoint (body); |
|
|
1095 { |
|
|
1096 llvm::Value *nelem |
|
|
1097 = builder.CreateExtractValue (fn.argument (builder, 0), 3); |
|
|
1098 llvm::Value *idx = fn.argument (builder, 1); |
|
|
1099 llvm::Value *ret = builder.CreateICmpULT (idx, nelem); |
|
|
1100 fn.do_return (builder, ret); |
|
|
1101 } |
|
|
1102 for_check_fn.add_overload (fn); |
|
|
1103 |
|
|
1104 // index variabe for for loop |
|
|
1105 for_index_fn.stash_name ("for_index"); |
|
|
1106 |
|
|
1107 fn = create_function (jit_convention::internal, "octave_jit_for_range_idx", |
|
|
1108 scalar, range, index); |
|
|
1109 body = fn.new_block (); |
|
|
1110 builder.SetInsertPoint (body); |
|
|
1111 { |
|
|
1112 llvm::Value *idx = fn.argument (builder, 1); |
|
|
1113 llvm::Value *didx = builder.CreateSIToFP (idx, scalar_t); |
|
|
1114 llvm::Value *rng = fn.argument (builder, 0); |
|
|
1115 llvm::Value *base = builder.CreateExtractValue (rng, 0); |
|
|
1116 llvm::Value *inc = builder.CreateExtractValue (rng, 2); |
|
|
1117 |
|
|
1118 llvm::Value *ret = builder.CreateFMul (didx, inc); |
|
|
1119 ret = builder.CreateFAdd (base, ret); |
|
|
1120 fn.do_return (builder, ret); |
|
|
1121 } |
|
|
1122 for_index_fn.add_overload (fn); |
|
|
1123 |
|
|
1124 // logically true |
|
|
1125 logically_true_fn.stash_name ("logically_true"); |
|
|
1126 |
|
|
1127 jit_function gripe_nantl |
|
|
1128 = create_function (jit_convention::external, |
|
|
1129 "octave_jit_gripe_nan_to_logical_conversion", 0); |
|
|
1130 gripe_nantl.mark_can_error (); |
|
|
1131 |
|
|
1132 fn = create_function (jit_convention::internal, |
|
|
1133 "octave_jit_logically_true_scalar", boolean, scalar); |
|
|
1134 fn.mark_can_error (); |
|
|
1135 |
|
|
1136 body = fn.new_block (); |
|
|
1137 builder.SetInsertPoint (body); |
|
|
1138 { |
|
|
1139 llvm::BasicBlock *error_block = fn.new_block ("error"); |
|
|
1140 llvm::BasicBlock *normal_block = fn.new_block ("normal"); |
|
|
1141 |
|
|
1142 llvm::Value *check = builder.CreateFCmpUNE (fn.argument (builder, 0), |
|
|
1143 fn.argument (builder, 0)); |
|
|
1144 builder.CreateCondBr (check, error_block, normal_block); |
|
|
1145 |
|
|
1146 builder.SetInsertPoint (error_block); |
|
|
1147 gripe_nantl.call (builder); |
|
|
1148 builder.CreateBr (normal_block); |
|
|
1149 builder.SetInsertPoint (normal_block); |
|
|
1150 |
|
|
1151 llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); |
|
|
1152 llvm::Value *ret = builder.CreateFCmpONE (fn.argument (builder, 0), zero); |
|
|
1153 fn.do_return (builder, ret); |
|
|
1154 } |
|
|
1155 logically_true_fn.add_overload (fn); |
|
|
1156 |
|
|
1157 // logically_true boolean |
|
|
1158 fn = create_identity (boolean); |
|
|
1159 logically_true_fn.add_overload (fn); |
|
|
1160 |
|
|
1161 // make_range |
|
|
1162 // FIXME: May be benificial to implement all in LLVM |
|
|
1163 make_range_fn.stash_name ("make_range"); |
|
|
1164 jit_function compute_nelem |
|
|
1165 = create_function (jit_convention::external, "octave_jit_compute_nelem", |
|
|
1166 index, scalar, scalar, scalar); |
|
|
1167 |
|
|
1168 fn = create_function (jit_convention::internal, "octave_jit_make_range", |
|
|
1169 range, scalar, scalar, scalar); |
|
|
1170 body = fn.new_block (); |
|
|
1171 builder.SetInsertPoint (body); |
|
|
1172 { |
|
|
1173 llvm::Value *base = fn.argument (builder, 0); |
|
|
1174 llvm::Value *limit = fn.argument (builder, 1); |
|
|
1175 llvm::Value *inc = fn.argument (builder, 2); |
|
|
1176 llvm::Value *nelem = compute_nelem.call (builder, base, limit, inc); |
|
|
1177 |
|
|
1178 llvm::Value *dzero = llvm::ConstantFP::get (scalar_t, 0); |
|
|
1179 llvm::Value *izero = llvm::ConstantInt::get (index_t, 0); |
|
|
1180 llvm::Value *rng = llvm::ConstantStruct::get (range_t, dzero, dzero, dzero, |
|
|
1181 izero, NULL); |
|
|
1182 rng = builder.CreateInsertValue (rng, base, 0); |
|
|
1183 rng = builder.CreateInsertValue (rng, limit, 1); |
|
|
1184 rng = builder.CreateInsertValue (rng, inc, 2); |
|
|
1185 rng = builder.CreateInsertValue (rng, nelem, 3); |
|
|
1186 fn.do_return (builder, rng); |
|
|
1187 } |
|
|
1188 make_range_fn.add_overload (fn); |
|
|
1189 |
|
|
1190 // paren_subsref |
|
|
1191 jit_type *jit_int = intN (sizeof (int) * 8); |
|
|
1192 llvm::Type *int_t = jit_int->to_llvm (); |
|
|
1193 jit_function ginvalid_index |
|
|
1194 = create_function (jit_convention::external, "octave_jit_ginvalid_index", |
|
|
1195 0); |
|
|
1196 jit_function gindex_range = create_function (jit_convention::external, |
|
|
1197 "octave_jit_gindex_range", |
|
|
1198 0, jit_int, jit_int, index, |
|
|
1199 index); |
|
|
1200 |
|
|
1201 fn = create_function (jit_convention::internal, "()subsref", scalar, matrix, |
|
|
1202 scalar); |
|
|
1203 fn.mark_can_error (); |
|
|
1204 |
|
|
1205 body = fn.new_block (); |
|
|
1206 builder.SetInsertPoint (body); |
|
|
1207 { |
|
|
1208 llvm::Value *one = llvm::ConstantInt::get (index_t, 1); |
|
|
1209 llvm::Value *ione; |
|
|
1210 if (index_t == int_t) |
|
|
1211 ione = one; |
|
|
1212 else |
|
|
1213 ione = llvm::ConstantInt::get (int_t, 1); |
|
|
1214 |
|
|
1215 llvm::Value *undef = llvm::UndefValue::get (scalar_t); |
|
|
1216 llvm::Value *mat = fn.argument (builder, 0); |
|
|
1217 llvm::Value *idx = fn.argument (builder, 1); |
|
|
1218 |
|
|
1219 // convert index to scalar to integer, and check index >= 1 |
|
|
1220 llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); |
|
|
1221 llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); |
|
|
1222 llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); |
|
|
1223 llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one); |
|
|
1224 llvm::Value *cond = builder.CreateOr (cond0, cond1); |
|
|
1225 |
|
|
1226 llvm::BasicBlock *done = fn.new_block ("done"); |
|
|
1227 llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); |
|
|
1228 llvm::BasicBlock *normal = fn.new_block ("normal", done); |
|
|
1229 builder.CreateCondBr (cond, conv_error, normal); |
|
|
1230 |
|
|
1231 builder.SetInsertPoint (conv_error); |
|
|
1232 ginvalid_index.call (builder); |
|
|
1233 builder.CreateBr (done); |
|
|
1234 |
|
|
1235 builder.SetInsertPoint (normal); |
|
|
1236 llvm::Value *len = builder.CreateExtractValue (mat, |
|
|
1237 llvm::ArrayRef<unsigned> (2)); |
|
|
1238 cond = builder.CreateICmpSGT (int_idx, len); |
|
|
1239 |
|
|
1240 |
|
|
1241 llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); |
|
|
1242 llvm::BasicBlock *success = fn.new_block ("success", done); |
|
|
1243 builder.CreateCondBr (cond, bounds_error, success); |
|
|
1244 |
|
|
1245 builder.SetInsertPoint (bounds_error); |
|
|
1246 gindex_range.call (builder, ione, ione, int_idx, len); |
|
|
1247 builder.CreateBr (done); |
|
|
1248 |
|
|
1249 builder.SetInsertPoint (success); |
|
|
1250 llvm::Value *data = builder.CreateExtractValue (mat, |
|
|
1251 llvm::ArrayRef<unsigned> (1)); |
|
|
1252 llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); |
|
|
1253 llvm::Value *ret = builder.CreateLoad (gep); |
|
|
1254 builder.CreateBr (done); |
|
|
1255 |
|
|
1256 builder.SetInsertPoint (done); |
|
|
1257 |
|
|
1258 llvm::PHINode *merge = llvm::PHINode::Create (scalar_t, 3); |
|
|
1259 builder.Insert (merge); |
|
|
1260 merge->addIncoming (undef, conv_error); |
|
|
1261 merge->addIncoming (undef, bounds_error); |
|
|
1262 merge->addIncoming (ret, success); |
|
|
1263 fn.do_return (builder, merge); |
|
|
1264 } |
|
|
1265 paren_subsref_fn.add_overload (fn); |
|
|
1266 |
|
|
1267 // paren subsasgn |
|
|
1268 paren_subsasgn_fn.stash_name ("()subsasgn"); |
|
|
1269 |
|
|
1270 jit_function resize_paren_subsasgn |
|
|
1271 = create_function (jit_convention::external, |
|
|
1272 "octave_jit_paren_subsasgn_impl", matrix, index, scalar); |
|
|
1273 fn = create_function (jit_convention::internal, "octave_jit_paren_subsasgn", |
|
|
1274 matrix, matrix, scalar, scalar); |
|
|
1275 fn.mark_can_error (); |
|
|
1276 body = fn.new_block (); |
|
|
1277 builder.SetInsertPoint (body); |
|
|
1278 { |
|
|
1279 llvm::Value *one = llvm::ConstantInt::get (index_t, 1); |
|
|
1280 |
|
|
1281 llvm::Value *mat = fn.argument (builder, 0); |
|
|
1282 llvm::Value *idx = fn.argument (builder, 1); |
|
|
1283 llvm::Value *value = fn.argument (builder, 2); |
|
|
1284 |
|
|
1285 llvm::Value *int_idx = builder.CreateFPToSI (idx, index_t); |
|
|
1286 llvm::Value *check_idx = builder.CreateSIToFP (int_idx, scalar_t); |
|
|
1287 llvm::Value *cond0 = builder.CreateFCmpUNE (idx, check_idx); |
|
|
1288 llvm::Value *cond1 = builder.CreateICmpSLT (int_idx, one); |
|
|
1289 llvm::Value *cond = builder.CreateOr (cond0, cond1); |
|
|
1290 |
|
|
1291 llvm::BasicBlock *done = fn.new_block ("done"); |
|
|
1292 |
|
|
1293 llvm::BasicBlock *conv_error = fn.new_block ("conv_error", done); |
|
|
1294 llvm::BasicBlock *normal = fn.new_block ("normal", done); |
|
|
1295 builder.CreateCondBr (cond, conv_error, normal); |
|
|
1296 builder.SetInsertPoint (conv_error); |
|
|
1297 ginvalid_index.call (builder); |
|
|
1298 builder.CreateBr (done); |
|
|
1299 |
|
|
1300 builder.SetInsertPoint (normal); |
|
|
1301 llvm::Value *len = builder.CreateExtractValue (mat, |
|
|
1302 llvm::ArrayRef<unsigned> (2)); |
|
|
1303 cond0 = builder.CreateICmpSGT (int_idx, len); |
|
|
1304 |
|
|
1305 llvm::Value *rcount = builder.CreateExtractValue (mat, 0); |
|
|
1306 rcount = builder.CreateLoad (rcount); |
|
|
1307 cond1 = builder.CreateICmpSGT (rcount, one); |
|
|
1308 cond = builder.CreateOr (cond0, cond1); |
|
|
1309 |
|
|
1310 llvm::BasicBlock *bounds_error = fn.new_block ("bounds_error", done); |
|
|
1311 llvm::BasicBlock *success = fn.new_block ("success", done); |
|
|
1312 builder.CreateCondBr (cond, bounds_error, success); |
|
|
1313 |
|
|
1314 // resize on out of bounds access |
|
|
1315 builder.SetInsertPoint (bounds_error); |
|
|
1316 llvm::Value *resize_result = resize_paren_subsasgn.call (builder, int_idx, |
|
|
1317 value); |
|
|
1318 builder.CreateBr (done); |
|
|
1319 |
|
|
1320 builder.SetInsertPoint (success); |
|
|
1321 llvm::Value *data = builder.CreateExtractValue (mat, |
|
|
1322 llvm::ArrayRef<unsigned> (1)); |
|
|
1323 llvm::Value *gep = builder.CreateInBoundsGEP (data, int_idx); |
|
|
1324 builder.CreateStore (value, gep); |
|
|
1325 builder.CreateBr (done); |
|
|
1326 |
|
|
1327 builder.SetInsertPoint (done); |
|
|
1328 |
|
|
1329 llvm::PHINode *merge = llvm::PHINode::Create (matrix_t, 3); |
|
|
1330 builder.Insert (merge); |
|
|
1331 merge->addIncoming (mat, conv_error); |
|
|
1332 merge->addIncoming (resize_result, bounds_error); |
|
|
1333 merge->addIncoming (mat, success); |
|
|
1334 fn.do_return (builder, merge); |
|
|
1335 } |
|
|
1336 paren_subsasgn_fn.add_overload (fn); |
|
|
1337 |
|
|
1338 fn = create_function (jit_convention::external, |
|
|
1339 "octave_jit_paren_subsasgn_matrix_range", matrix, |
|
|
1340 matrix, range, scalar); |
|
|
1341 fn.mark_can_error (); |
|
|
1342 paren_subsasgn_fn.add_overload (fn); |
|
|
1343 |
|
|
1344 casts[any->type_id ()].stash_name ("(any)"); |
|
|
1345 casts[scalar->type_id ()].stash_name ("(scalar)"); |
|
|
1346 casts[complex->type_id ()].stash_name ("(complex)"); |
|
|
1347 casts[matrix->type_id ()].stash_name ("(matrix)"); |
|
|
1348 |
|
|
1349 // cast any <- matrix |
|
|
1350 fn = create_function (jit_convention::external, "octave_jit_cast_any_matrix", |
|
|
1351 any, matrix); |
|
|
1352 casts[any->type_id ()].add_overload (fn); |
|
|
1353 |
|
|
1354 // cast matrix <- any |
|
|
1355 fn = create_function (jit_convention::external, "octave_jit_cast_matrix_any", |
|
|
1356 matrix, any); |
|
|
1357 casts[matrix->type_id ()].add_overload (fn); |
|
|
1358 |
|
|
1359 // cast any <- scalar |
|
|
1360 fn = create_function (jit_convention::external, "octave_jit_cast_any_scalar", |
|
|
1361 any, scalar); |
|
|
1362 casts[any->type_id ()].add_overload (fn); |
|
|
1363 |
|
|
1364 // cast scalar <- any |
|
|
1365 fn = create_function (jit_convention::external, "octave_jit_cast_scalar_any", |
|
|
1366 scalar, any); |
|
|
1367 casts[scalar->type_id ()].add_overload (fn); |
|
|
1368 |
|
|
1369 // cast any <- complex |
|
|
1370 fn = create_function (jit_convention::external, "octave_jit_cast_any_complex", |
|
|
1371 any, complex); |
|
|
1372 casts[any->type_id ()].add_overload (fn); |
|
|
1373 |
|
|
1374 // cast complex <- any |
|
|
1375 fn = create_function (jit_convention::external, "octave_jit_cast_complex_any", |
|
|
1376 complex, any); |
|
|
1377 casts[complex->type_id ()].add_overload (fn); |
|
|
1378 |
|
|
1379 // cast complex <- scalar |
|
|
1380 fn = create_function (jit_convention::internal, |
|
|
1381 "octave_jit_cast_complex_scalar", complex, scalar); |
|
|
1382 body = fn.new_block (); |
|
|
1383 builder.SetInsertPoint (body); |
|
|
1384 { |
|
|
1385 llvm::Value *zero = llvm::ConstantFP::get (scalar_t, 0); |
|
|
1386 fn.do_return (builder, complex_new (fn.argument (builder, 0), zero)); |
|
|
1387 } |
|
|
1388 casts[complex->type_id ()].add_overload (fn); |
|
|
1389 |
|
|
1390 // cast scalar <- complex |
|
|
1391 fn = create_function (jit_convention::internal, |
|
|
1392 "octave_jit_cast_scalar_complex", scalar, complex); |
|
|
1393 body = fn.new_block (); |
|
|
1394 builder.SetInsertPoint (body); |
|
|
1395 fn.do_return (builder, complex_real (fn.argument (builder, 0))); |
|
|
1396 casts[scalar->type_id ()].add_overload (fn); |
|
|
1397 |
|
|
1398 // cast any <- any |
|
|
1399 fn = create_identity (any); |
|
|
1400 casts[any->type_id ()].add_overload (fn); |
|
|
1401 |
|
|
1402 // cast scalar <- scalar |
|
|
1403 fn = create_identity (scalar); |
|
|
1404 casts[scalar->type_id ()].add_overload (fn); |
|
|
1405 |
|
|
1406 // cast complex <- complex |
|
|
1407 fn = create_identity (complex); |
|
|
1408 casts[complex->type_id ()].add_overload (fn); |
|
|
1409 |
|
|
1410 // -------------------- builtin functions -------------------- |
|
|
1411 add_builtin ("#unknown_function"); |
|
|
1412 unknown_function = builtins["#unknown_function"]; |
|
|
1413 |
|
|
1414 add_builtin ("sin"); |
|
|
1415 register_intrinsic ("sin", llvm::Intrinsic::sin, scalar, scalar); |
|
|
1416 register_generic ("sin", matrix, matrix); |
|
|
1417 |
|
|
1418 add_builtin ("cos"); |
|
|
1419 register_intrinsic ("cos", llvm::Intrinsic::cos, scalar, scalar); |
|
|
1420 register_generic ("cos", matrix, matrix); |
|
|
1421 |
|
|
1422 add_builtin ("exp"); |
|
|
1423 register_intrinsic ("exp", llvm::Intrinsic::cos, scalar, scalar); |
|
|
1424 register_generic ("exp", matrix, matrix); |
|
|
1425 |
|
|
1426 casts.resize (next_id + 1); |
|
|
1427 jit_function any_id = create_identity (any); |
|
|
1428 jit_function release_any = get_release (any); |
|
|
1429 std::vector<jit_type *> args; |
|
|
1430 args.resize (1); |
|
|
1431 |
|
|
1432 for (std::map<std::string, jit_type *>::iterator iter = builtins.begin (); |
|
|
1433 iter != builtins.end (); ++iter) |
|
|
1434 { |
|
|
1435 jit_type *btype = iter->second; |
|
|
1436 args[0] = btype; |
|
|
1437 |
|
|
1438 release_fn.add_overload (jit_function (release_any, 0, args)); |
|
|
1439 casts[any->type_id ()].add_overload (jit_function (any_id, any, args)); |
|
|
1440 |
|
|
1441 args[0] = any; |
|
|
1442 casts[btype->type_id ()].add_overload (jit_function (any_id, btype, |
|
|
1443 args)); |
|
|
1444 } |
|
|
1445 } |
|
|
1446 |
|
|
1447 void |
|
|
1448 jit_typeinfo::add_print (jit_type *ty) |
|
|
1449 { |
|
|
1450 std::stringstream name; |
|
|
1451 name << "octave_jit_print_" << ty->name (); |
|
|
1452 jit_function fn = create_function (jit_convention::external, name.str (), 0, |
|
|
1453 intN (8), ty); |
|
|
1454 print_fn.add_overload (fn); |
|
|
1455 } |
|
|
1456 |
|
|
1457 // FIXME: cp between add_binary_op, add_binary_icmp, and add_binary_fcmp |
|
|
1458 void |
|
|
1459 jit_typeinfo::add_binary_op (jit_type *ty, int op, int llvm_op) |
|
|
1460 { |
|
|
1461 std::stringstream fname; |
|
|
1462 octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); |
|
|
1463 fname << "octave_jit_" << octave_value::binary_op_as_string (ov_op) |
|
|
1464 << "_" << ty->name (); |
|
|
1465 |
|
|
1466 jit_function fn = create_function (jit_convention::internal, fname.str (), |
|
|
1467 ty, ty, ty); |
|
|
1468 llvm::BasicBlock *block = fn.new_block (); |
|
|
1469 builder.SetInsertPoint (block); |
|
|
1470 llvm::Instruction::BinaryOps temp |
|
|
1471 = static_cast<llvm::Instruction::BinaryOps>(llvm_op); |
|
|
1472 |
|
|
1473 llvm::Value *ret = builder.CreateBinOp (temp, fn.argument (builder, 0), |
|
|
1474 fn.argument (builder, 1)); |
|
|
1475 fn.do_return (builder, ret); |
|
|
1476 binary_ops[op].add_overload (fn); |
|
|
1477 } |
|
|
1478 |
|
|
1479 void |
|
|
1480 jit_typeinfo::add_binary_icmp (jit_type *ty, int op, int llvm_op) |
|
|
1481 { |
|
|
1482 std::stringstream fname; |
|
|
1483 octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); |
|
|
1484 fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) |
|
|
1485 << "_" << ty->name (); |
|
|
1486 |
|
|
1487 jit_function fn = create_function (jit_convention::internal, fname.str (), |
|
|
1488 boolean, ty, ty); |
|
|
1489 llvm::BasicBlock *block = fn.new_block (); |
|
|
1490 builder.SetInsertPoint (block); |
|
|
1491 llvm::CmpInst::Predicate temp |
|
|
1492 = static_cast<llvm::CmpInst::Predicate>(llvm_op); |
|
|
1493 llvm::Value *ret = builder.CreateICmp (temp, fn.argument (builder, 0), |
|
|
1494 fn.argument (builder, 1)); |
|
|
1495 fn.do_return (builder, ret); |
|
|
1496 binary_ops[op].add_overload (fn); |
|
|
1497 } |
|
|
1498 |
|
|
1499 void |
|
|
1500 jit_typeinfo::add_binary_fcmp (jit_type *ty, int op, int llvm_op) |
|
|
1501 { |
|
|
1502 std::stringstream fname; |
|
|
1503 octave_value::binary_op ov_op = static_cast<octave_value::binary_op>(op); |
|
|
1504 fname << "octave_jit" << octave_value::binary_op_as_string (ov_op) |
|
|
1505 << "_" << ty->name (); |
|
|
1506 |
|
|
1507 jit_function fn = create_function (jit_convention::internal, fname.str (), |
|
|
1508 boolean, ty, ty); |
|
|
1509 llvm::BasicBlock *block = fn.new_block (); |
|
|
1510 builder.SetInsertPoint (block); |
|
|
1511 llvm::CmpInst::Predicate temp |
|
|
1512 = static_cast<llvm::CmpInst::Predicate>(llvm_op); |
|
|
1513 llvm::Value *ret = builder.CreateFCmp (temp, fn.argument (builder, 0), |
|
|
1514 fn.argument (builder, 1)); |
|
|
1515 fn.do_return (builder, ret); |
|
|
1516 binary_ops[op].add_overload (fn); |
|
|
1517 } |
|
|
1518 |
|
|
1519 jit_function |
|
|
1520 jit_typeinfo::create_function (jit_convention::type cc, const llvm::Twine& name, |
|
|
1521 jit_type *ret, |
|
|
1522 const std::vector<jit_type *>& args) |
|
|
1523 { |
|
|
1524 jit_function result (module, cc, name, ret, args); |
|
|
1525 return result; |
|
|
1526 } |
|
|
1527 |
|
|
1528 jit_function |
|
|
1529 jit_typeinfo::create_identity (jit_type *type) |
|
|
1530 { |
|
|
1531 size_t id = type->type_id (); |
|
|
1532 if (id >= identities.size ()) |
|
|
1533 identities.resize (id + 1); |
|
|
1534 |
|
|
1535 if (! identities[id].valid ()) |
|
|
1536 { |
|
|
1537 jit_function fn = create_function (jit_convention::internal, "id", type, |
|
|
1538 type); |
|
|
1539 llvm::BasicBlock *body = fn.new_block (); |
|
|
1540 builder.SetInsertPoint (body); |
|
|
1541 fn.do_return (builder, fn.argument (builder, 0)); |
|
|
1542 return identities[id] = fn; |
|
|
1543 } |
|
|
1544 |
|
|
1545 return identities[id]; |
|
|
1546 } |
|
|
1547 |
|
|
1548 llvm::Value * |
|
|
1549 jit_typeinfo::do_insert_error_check (llvm::IRBuilderD& builder) |
|
|
1550 { |
|
|
1551 return builder.CreateLoad (lerror_state); |
|
|
1552 } |
|
|
1553 |
|
|
1554 void |
|
|
1555 jit_typeinfo::add_builtin (const std::string& name) |
|
|
1556 { |
|
|
1557 jit_type *btype = new_type (name, any, any->to_llvm ()); |
|
|
1558 builtins[name] = btype; |
|
|
1559 |
|
|
1560 octave_builtin *ov_builtin = find_builtin (name); |
|
|
1561 if (ov_builtin) |
|
|
1562 ov_builtin->stash_jit (*btype); |
|
|
1563 } |
|
|
1564 |
|
|
1565 void |
|
|
1566 jit_typeinfo::register_intrinsic (const std::string& name, size_t iid, |
|
|
1567 jit_type *result, |
|
|
1568 const std::vector<jit_type *>& args) |
|
|
1569 { |
|
|
1570 jit_type *builtin_type = builtins[name]; |
|
|
1571 size_t nargs = args.size (); |
|
|
1572 llvm::SmallVector<llvm::Type *, 5> llvm_args (nargs); |
|
|
1573 for (size_t i = 0; i < nargs; ++i) |
|
|
1574 llvm_args[i] = args[i]->to_llvm (); |
|
|
1575 |
|
|
1576 llvm::Intrinsic::ID id = static_cast<llvm::Intrinsic::ID> (iid); |
|
|
1577 llvm::Function *ifun = llvm::Intrinsic::getDeclaration (module, id, |
|
|
1578 llvm_args); |
|
|
1579 std::stringstream fn_name; |
|
|
1580 fn_name << "octave_jit_" << name; |
|
|
1581 |
|
|
1582 std::vector<jit_type *> args1 (nargs + 1); |
|
|
1583 args1[0] = builtin_type; |
|
|
1584 std::copy (args.begin (), args.end (), args1.begin () + 1); |
|
|
1585 |
|
|
1586 // The first argument will be the Octave function, but we already know that |
|
|
1587 // the function call is the equivalent of the intrinsic, so we ignore it and |
|
|
1588 // call the intrinsic with the remaining arguments. |
|
|
1589 jit_function fn = create_function (jit_convention::internal, fn_name.str (), |
|
|
1590 result, args1); |
|
|
1591 llvm::BasicBlock *body = fn.new_block (); |
|
|
1592 builder.SetInsertPoint (body); |
|
|
1593 |
|
|
1594 llvm::SmallVector<llvm::Value *, 5> fargs (nargs); |
|
|
1595 for (size_t i = 0; i < nargs; ++i) |
|
|
1596 fargs[i] = fn.argument (builder, i + 1); |
|
|
1597 |
|
|
1598 llvm::Value *ret = builder.CreateCall (ifun, fargs); |
|
|
1599 fn.do_return (builder, ret); |
|
|
1600 paren_subsref_fn.add_overload (fn); |
|
|
1601 } |
|
|
1602 |
|
|
1603 octave_builtin * |
|
|
1604 jit_typeinfo::find_builtin (const std::string& name) |
|
|
1605 { |
|
|
1606 // FIXME: Finalize what we want to store in octave_builtin, then add functions |
|
|
1607 // to access these values in octave_value |
|
|
1608 octave_value ov_builtin = symbol_table::find (name); |
|
|
1609 return dynamic_cast<octave_builtin *> (ov_builtin.internal_rep ()); |
|
|
1610 } |
|
|
1611 |
|
|
1612 void |
|
|
1613 jit_typeinfo::register_generic (const std::string&, jit_type *, |
|
|
1614 const std::vector<jit_type *>&) |
|
|
1615 { |
|
|
1616 // FIXME: Implement |
|
|
1617 } |
|
|
1618 |
|
|
1619 jit_function |
|
|
1620 jit_typeinfo::mirror_binary (const jit_function& fn) |
|
|
1621 { |
|
|
1622 jit_function ret = create_function (jit_convention::internal, |
|
|
1623 fn.name () + "_reverse", |
|
|
1624 fn.result (), fn.argument_type (1), |
|
|
1625 fn.argument_type (0)); |
|
|
1626 if (fn.can_error ()) |
|
|
1627 ret.mark_can_error (); |
|
|
1628 |
|
|
1629 llvm::BasicBlock *body = ret.new_block (); |
|
|
1630 builder.SetInsertPoint (body); |
|
|
1631 llvm::Value *result = fn.call (builder, ret.argument (builder, 1), |
|
|
1632 ret.argument (builder, 0)); |
|
|
1633 if (ret.result ()) |
|
|
1634 ret.do_return (builder, result); |
|
|
1635 else |
|
|
1636 ret.do_return (builder); |
|
|
1637 |
|
|
1638 return ret; |
|
|
1639 } |
|
|
1640 |
|
|
1641 llvm::Value * |
|
|
1642 jit_typeinfo::pack_complex (llvm::IRBuilderD& bld, llvm::Value *cplx) |
|
|
1643 { |
|
|
1644 llvm::Type *complex_ret = instance->complex_ret; |
|
|
1645 llvm::Value *real = bld.CreateExtractElement (cplx, bld.getInt32 (0)); |
|
|
1646 llvm::Value *imag = bld.CreateExtractElement (cplx, bld.getInt32 (1)); |
|
|
1647 llvm::Value *ret = llvm::UndefValue::get (complex_ret); |
|
|
1648 ret = bld.CreateInsertValue (ret, real, 0); |
|
|
1649 return bld.CreateInsertValue (ret, imag, 1); |
|
|
1650 } |
|
|
1651 |
|
|
1652 llvm::Value * |
|
|
1653 jit_typeinfo::unpack_complex (llvm::IRBuilderD& bld, llvm::Value *result) |
|
|
1654 { |
|
|
1655 llvm::Type *complex_t = get_complex ()->to_llvm (); |
|
|
1656 llvm::Value *real = bld.CreateExtractValue (result, 0); |
|
|
1657 llvm::Value *imag = bld.CreateExtractValue (result, 1); |
|
|
1658 llvm::Value *ret = llvm::UndefValue::get (complex_t); |
|
|
1659 ret = bld.CreateInsertElement (ret, real, bld.getInt32 (0)); |
|
|
1660 return bld.CreateInsertElement (ret, imag, bld.getInt32 (1)); |
|
|
1661 } |
|
|
1662 |
|
|
1663 llvm::Value * |
|
|
1664 jit_typeinfo::complex_real (llvm::Value *cx) |
|
|
1665 { |
|
|
1666 return builder.CreateExtractElement (cx, builder.getInt32 (0)); |
|
|
1667 } |
|
|
1668 |
|
|
1669 llvm::Value * |
|
|
1670 jit_typeinfo::complex_real (llvm::Value *cx, llvm::Value *real) |
|
|
1671 { |
|
|
1672 return builder.CreateInsertElement (cx, real, builder.getInt32 (0)); |
|
|
1673 } |
|
|
1674 |
|
|
1675 llvm::Value * |
|
|
1676 jit_typeinfo::complex_imag (llvm::Value *cx) |
|
|
1677 { |
|
|
1678 return builder.CreateExtractElement (cx, builder.getInt32 (1)); |
|
|
1679 } |
|
|
1680 |
|
|
1681 llvm::Value * |
|
|
1682 jit_typeinfo::complex_imag (llvm::Value *cx, llvm::Value *imag) |
|
|
1683 { |
|
|
1684 return builder.CreateInsertElement (cx, imag, builder.getInt32 (1)); |
|
|
1685 } |
|
|
1686 |
|
|
1687 llvm::Value * |
|
|
1688 jit_typeinfo::complex_new (llvm::Value *real, llvm::Value *imag) |
|
|
1689 { |
|
|
1690 llvm::Value *ret = llvm::UndefValue::get (complex->to_llvm ()); |
|
|
1691 ret = complex_real (ret, real); |
|
|
1692 return complex_imag (ret, imag); |
|
|
1693 } |
|
|
1694 |
|
|
1695 void |
|
|
1696 jit_typeinfo::create_int (size_t nbits) |
|
|
1697 { |
|
|
1698 std::stringstream tname; |
|
|
1699 tname << "int" << nbits; |
|
|
1700 ints[nbits] = new_type (tname.str (), any, llvm::Type::getIntNTy (context, |
|
|
1701 nbits)); |
|
|
1702 } |
|
|
1703 |
|
|
1704 jit_type * |
|
|
1705 jit_typeinfo::intN (size_t nbits) const |
|
|
1706 { |
|
|
1707 std::map<size_t, jit_type *>::const_iterator iter = ints.find (nbits); |
|
|
1708 if (iter != ints.end ()) |
|
|
1709 return iter->second; |
|
|
1710 |
|
|
1711 throw jit_fail_exception ("No such integer type"); |
|
|
1712 } |
|
|
1713 |
|
|
1714 jit_type * |
|
|
1715 jit_typeinfo::do_type_of (const octave_value &ov) const |
|
|
1716 { |
|
|
1717 if (ov.is_function ()) |
|
|
1718 { |
|
|
1719 // FIXME: This is ugly, we need to finalize how we want to to this, then |
|
|
1720 // have octave_value fully support the needed functionality |
|
|
1721 octave_builtin *builtin |
|
|
1722 = dynamic_cast<octave_builtin *> (ov.internal_rep ()); |
|
|
1723 return builtin && builtin->to_jit () ? builtin->to_jit () |
|
|
1724 : unknown_function; |
|
|
1725 } |
|
|
1726 |
|
|
1727 if (ov.is_range ()) |
|
|
1728 return get_range (); |
|
|
1729 |
|
|
1730 if (ov.is_double_type ()) |
|
|
1731 { |
|
|
1732 if (ov.is_real_scalar ()) |
|
|
1733 return get_scalar (); |
|
|
1734 |
|
|
1735 if (ov.is_matrix_type ()) |
|
|
1736 return get_matrix (); |
|
|
1737 } |
|
|
1738 |
|
|
1739 if (ov.is_complex_scalar ()) |
|
|
1740 return get_complex (); |
|
|
1741 |
|
|
1742 return get_any (); |
|
|
1743 } |
|
|
1744 |
|
|
1745 jit_type* |
|
|
1746 jit_typeinfo::new_type (const std::string& name, jit_type *parent, |
|
|
1747 llvm::Type *llvm_type) |
|
|
1748 { |
|
|
1749 jit_type *ret = new jit_type (name, parent, llvm_type, next_id++); |
|
|
1750 id_to_type.push_back (ret); |
|
|
1751 return ret; |
|
|
1752 } |
|
|
1753 |
|
|
1754 #endif |
