Merge pull request #61 from pypr/fix-show-source

Many fixes and improvements

Author: prabhuramachandran

README.rst

@@ -1,4 +1,4 @@
-ComPyle: execute a subset of Python on HPC platforms
+Compyle: execute a subset of Python on HPC platforms
 ======================================================
 
 |Travis Status| |Appveyor Status| |Coverage Status| |Documentation Status|
@@ -14,38 +14,48 @@ ComPyle: execute a subset of Python on HPC platforms
 .. |Coverage Status| image:: https://codecov.io/gh/pypr/compyle/branch/master/graph/badge.svg
     :target: https://codecov.io/gh/pypr/compyle
 
-ComPyle allows users to execute a restricted subset of Python (almost similar
-to C) on a variety of HPC platforms. Currently we support multi-core execution
-using Cython, and OpenCL and CUDA for GPU devices.
+Compyle allows users to execute a restricted subset of Python (almost similar
+to C) on a variety of HPC platforms. Currently we support multi-core CPU
+execution using Cython, and for GPU devices we use OpenCL or CUDA.
 
 Users start with code implemented in a very restricted Python syntax, this code
 is then automatically transpiled, compiled and executed to run on either one CPU
-core, or multiple CPU cores (via OpenMP_) or on a GPU. ComPyle offers
+core, or multiple CPU cores (via OpenMP_) or on a GPU. Compyle offers
 source-to-source transpilation, making it a very convenient tool for writing HPC
 libraries.
 
-Some simple yet powerful parallel utilities are provided which can allow you to
-solve a remarkably large number of interesting HPC problems.
+Some simple yet powerful parallel utilities are provided which can allow you
+to solve a remarkably large number of interesting HPC problems. Compyle also
+features JIT transpilation making it easy to use.
 
-ComPyle also features JIT transpilation if you wish making it easy to use.
+Documentation and learning material is also available in the form of:
 
-Documentation is available at: https://compyle.readthedocs.io
+- Documentation at: https://compyle.readthedocs.io
 
-While ComPyle seems simple it is not a toy and is used heavily by the PySPH_
-project where ComPyle has its origins.
+- An introduction to compyle in the context of writing a parallel molecular
+  dynamics simulator is in our `SciPy 2020 paper
+  <http://conference.scipy.org/proceedings/scipy2020/compyle_pr_ab.html>`_.
+
+- `Compyle poster presentation <https://docs.google.com/presentation/d/1LS9XO5pQXz8G5d27RP5oWLFxUA-Fr5OvfVUGsgg86TQ/edit#slide=id.p>`_
+
+- You may also try Compyle online for free on a `Google Colab notebook`_.
+
+While Compyle seems simple it is not a toy and is used heavily by the PySPH_
+project where Compyle has its origins.
 
 .. _PySPH: https://github.com/pypr/pysph
+.. _Google Colab notebook: https://colab.research.google.com/drive/1SGRiArYXV1LEkZtUeg9j0qQ21MDqQR2U?usp=sharing
 
 
 Installation
 -------------
 
-ComPyle is itself pure Python but depends on numpy_ and requires either Cython_
-or PyOpenCL_ or PyCUDA_ along with the respective backends of a C/C++ compiler,
-OpenCL and CUDA. If you are only going to execute code on a CPU then all you
-need is Cython.
+Compyle is itself largely pure Python but depends on numpy_ and requires
+either Cython_ or PyOpenCL_ or PyCUDA_ along with the respective backends of a
+C/C++ compiler, OpenCL and CUDA. If you are only going to execute code on a
+CPU then all you need is Cython.
 
-You should be able to install ComPyle by doing::
+You should be able to install Compyle by doing::
 
   $ pip install compyle
 
@@ -95,3 +105,5 @@ Examples
 
 Some simple examples and benchmarks are available in the `examples
 <https://github.com/pypr/compyle/tree/master/examples>`_ directory.
+
+You may also run these examples on the `Google Colab notebook`_

compyle/api.py

@@ -9,7 +9,7 @@
 from .extern import Extern
 from .low_level import Kernel, LocalMem, Cython
 from .parallel import (
-    Elementwise, Reduction, elementwise
+    Elementwise, Reduction, Scan, elementwise
 )
 from .translator import (
     CConverter, CStructHelper, OpenCLConverter, detect_type, ocl_detect_type,

compyle/jit.py

@@ -293,6 +293,8 @@ def __init__(self, func, backend=None):
         self.func = func
         self._config = get_config()
         self.cython_gen = CythonGenerator()
+        self.source = '# Code jitted, call the function to generate the code.'
+        self.all_source = self.source
         if backend == 'opencl':
             from .opencl import get_context, get_queue
             self.queue = get_queue()
@@ -369,6 +371,8 @@ def __init__(self, reduce_expr, map_func=None, dtype_out=np.float64,
             self.neutral = neutral
         self._config = get_config()
         self.cython_gen = CythonGenerator()
+        self.source = '# Code jitted, call the function to generate the code.'
+        self.all_source = self.source
         if backend == 'opencl':
             from .opencl import get_context, get_queue
             self.queue = get_queue()
@@ -450,6 +454,8 @@ def __init__(self, input=None, output=None, scan_expr="a+b",
         else:
             self.neutral = neutral
         self._config = get_config()
+        self.source = '# Code jitted, call the function to generate the code.'
+        self.all_source = self.source
         self.cython_gen = CythonGenerator()
         if backend == 'opencl':
             from .opencl import get_context, get_queue

compyle/parallel.py

@@ -11,7 +11,6 @@
 
 from mako.template import Template
 import numpy as np
-import time
 
 from .config import get_config
 from .profile import profile
@@ -412,6 +411,10 @@ def __init__(self, func, backend=None):
         self._config = get_config()
         self.cython_gen = CythonGenerator()
         self.queue = None
+        # This is the source generated for the user code.
+        self.source = '# Source not yet generated.'
+        # This is all the source code used for the elementwise.
+        self.all_source = '# Source not yet generated.'
         self.c_func = self._generate()
 
     def _generate(self, declarations=None):
@@ -433,8 +436,12 @@ def _generate(self, declarations=None):
                     self.func, 'is_serial', False),
                 get_parallel_range=get_parallel_range
             )
+            # This is the user code source.
+            self.source = self.tp.get_code()
             self.tp.add_code(src)
             self.tp.compile()
+            # All the source code for the elementwise
+            self.all_source = self.tp.source
             return getattr(self.tp.mod, 'py_' + self.name[7:])
         elif self.backend == 'opencl':
             py_data, c_data = self.cython_gen.get_func_signature(self.func)
@@ -462,6 +469,10 @@ def _generate(self, declarations=None):
                 operation=expr,
                 preamble="\n".join([cluda_preamble, preamble])
             )
+            # only code we generate is saved here.
+            self.source = "\n".join([cluda_preamble, preamble])
+            all_source = knl.get_kernel(False)[0].program.source
+            self.all_source = all_source or self.source
             return knl
         elif self.backend == 'cuda':
             py_data, c_data = self.cython_gen.get_func_signature(self.func)
@@ -487,6 +498,10 @@ def _generate(self, declarations=None):
                 operation=expr,
                 preamble="\n".join([cluda_preamble, preamble])
             )
+            # only code we generate is saved here.
+            self.source = cluda_preamble + preamble
+            # FIXME: it is difficult to get the sources from pycuda.
+            self.all_source = self.source
             return knl
 
     def _correct_opencl_address_space(self, c_data):
@@ -551,6 +566,9 @@ def __init__(self, func, backend=None):
     def __getattr__(self, name):
         return getattr(self.elementwise, name)
 
+    def __dir__(self):
+        return sorted(dir(self.elementwise) + ['elementwise'])
+
     def __call__(self, *args, **kwargs):
         self.elementwise(*args, **kwargs)
 
@@ -588,6 +606,10 @@ def __init__(self, reduce_expr, map_func=None, dtype_out=np.float64,
         self._config = get_config()
         self.cython_gen = CythonGenerator()
         self.queue = None
+        # This is the source generated for the user code.
+        self.source = '# Source not yet generated.'
+        # This is all the source code used.
+        self.all_source = '# Source not yet generated.'
         self.c_func = self._generate()
 
     def _generate(self, declarations=None):
@@ -621,8 +643,11 @@ def _generate(self, declarations=None):
                 openmp=self._config.use_openmp,
                 get_parallel_range=get_parallel_range
             )
+            # This is the user code source.
+            self.source = self.tp.get_code()
             self.tp.add_code(src)
             self.tp.compile()
+            self.all_source = self.tp.source
             return getattr(self.tp.mod, 'py_' + self.name)
         elif self.backend == 'opencl':
             if self.func is not None:
@@ -661,6 +686,17 @@ def _generate(self, declarations=None):
                 arguments=arguments,
                 preamble="\n".join([cluda_preamble, preamble])
             )
+            # only code we generate is saved here.
+            self.source = "\n".join([cluda_preamble, preamble])
+            if knl.stage_1_inf.source:
+                self.all_source = "\n".join([
+                    "// ------ stage 1 -----",
+                    knl.stage_1_inf.source,
+                    "// ------ stage 2 -----",
+                    knl.stage_2_inf.source,
+                ])
+            else:
+                self.all_source = self.source
             return knl
         elif self.backend == 'cuda':
             if self.func is not None:
@@ -697,6 +733,10 @@ def _generate(self, declarations=None):
                 arguments=arguments,
                 preamble="\n".join([cluda_preamble, preamble])
             )
+            # only code we generate is saved here.
+            self.source = cluda_preamble + preamble
+            # FIXME: it is difficult to get the sources from pycuda.
+            self.all_source = self.source
             return knl
 
     def _correct_return_type(self, c_data):
@@ -780,6 +820,9 @@ def __init__(self, reduce_expr, map_func=None, dtype_out=np.float64,
                                           neutral=neutral,
                                           backend=backend)
 
+    def __dir__(self):
+        return sorted(dir(self.reduction) + ['reduction'])
+
     def __getattr__(self, name):
         return getattr(self.reduction, name)
 
@@ -812,6 +855,10 @@ def __init__(self, input=None, output=None, scan_expr="a+b",
         else:
             self.neutral = neutral
         self._config = get_config()
+        # This is the source generated for the user code.
+        self.source = '# Source not yet generated.'
+        # This is all the source code used for the elementwise.
+        self.all_source = '# Source not yet generated.'
         self.cython_gen = CythonGenerator()
         self.queue = None
         self.c_func = self._generate()
@@ -895,7 +942,6 @@ def _append_cython_arg_data(self, all_py_data, all_c_data, py_data,
             all_c_data[1].extend(self._filter_ignored(c_data[1], select))
 
     def _generate_cython_code(self, declarations=None):
-        name = self.name
         all_py_data = [[], []]
         all_c_data = [[], []]
 
@@ -911,7 +957,9 @@ def _generate_cython_code(self, declarations=None):
         # Process segment function
         use_segment = True if self.is_segment_func is not None else False
         py_data, c_data, segment_expr = self._wrap_cython_code(
-            self.is_segment_func, func_type='segment', declarations=declarations)
+            self.is_segment_func, func_type='segment',
+            declarations=declarations
+        )
         self._append_cython_arg_data(all_py_data, all_c_data, py_data, c_data)
 
         # Process output expression
@@ -963,8 +1011,10 @@ def _generate_cython_code(self, declarations=None):
             is_segment_start_expr=segment_expr,
             complex_map=self.complex_map
         )
+        self.source = self.tp.get_code()
         self.tp.add_code(src)
         self.tp.compile()
+        self.all_source = self.tp.source
         return getattr(self.tp.mod, 'py_' + self.name)
 
     def _wrap_ocl_function(self, func, func_type=None, declarations=None):
@@ -1053,6 +1103,18 @@ def _generate_opencl_kernel(self, declarations=None):
             is_segment_start_expr=segment_expr,
             preamble=preamble
         )
+        self.source = preamble
+        if knl.first_level_scan_info.kernel.program.source:
+            self.all_source = '\n'.join([
+                '// ----- Level 1 ------',
+                knl.first_level_scan_info.kernel.program.source,
+                '// ----- Level 2 ------',
+                knl.second_level_scan_info.kernel.program.source,
+                '// ----- Final output ------',
+                knl.final_update_info.kernel.program.source,
+            ])
+        else:
+            self.all_source = self.source
         return knl
 
     def _generate_cuda_kernel(self, declarations=None):
@@ -1073,6 +1135,9 @@ def _generate_cuda_kernel(self, declarations=None):
             is_segment_start_expr=segment_expr,
             preamble=preamble
         )
+        self.source = preamble
+        # FIXME: Difficult to get the pycuda sources
+        self.all_source = self.source
         return knl
 
     def _add_address_space(self, arg):
@@ -1113,11 +1178,13 @@ def _massage_arg(self, x):
     def __call__(self, **kwargs):
         c_args_dict = {k: self._massage_arg(x) for k, x in kwargs.items()}
         if self._get_backend_key() in self.output_func.arg_keys:
-            output_arg_keys = self.output_func.arg_keys[self._get_backend_key()]
+            output_arg_keys = self.output_func.arg_keys[
+                self._get_backend_key()
+            ]
         else:
             raise ValueError("No kernel arguments found for backend = %s, "
-                              "use_openmp = %s, use_double = %s" %
-                                      self._get_backend_key())
+                             "use_openmp = %s, use_double = %s" %
+                             self._get_backend_key())
 
         if self.backend == 'cython':
             size = len(c_args_dict[output_arg_keys[1]])
@@ -1165,6 +1232,9 @@ def __init__(self, input=None, output=None, scan_expr="a+b",
                                 complex_map=complex_map,
                                 backend=backend)
 
+    def __dir__(self):
+        return sorted(dir(self.scan) + ['scan'])
+
     def __getattr__(self, name):
         return getattr(self.scan, name)