Merge pull request #615 from skrah/separate_datashape

Separate ndt from array and callables.

Author: izaid

.clang-format

@@ -6,4 +6,4 @@ BreakBeforeBraces: Stroustrup
 Cpp11BracedListStyle: true
 NamespaceIndentation: Inner
 AlwaysBreakTemplateDeclarations: true
-
+ColumnLimit: 120

CMakeLists.txt

@@ -157,26 +157,33 @@ cython_add_module(dynd.nd.array dynd.nd.array_pyx True
                   dynd/src/array_as_pep3118.cpp
                   dynd/src/array_as_numpy.cpp
                   dynd/src/array_from_py.cpp
-                  dynd/src/array_from_py_typededuction.cpp
                   dynd/src/assign.cpp
-                  dynd/src/conversions.cpp
+                  dynd/src/array_conversions.cpp
                   dynd/src/copy_from_numpy_arrfunc.cpp
                   dynd/src/init.cpp
                   dynd/src/functional.cpp
                   dynd/src/numpy_interop.cpp
+                  dynd/src/type_conversions.cpp
+                  dynd/src/type_deduction.cpp
                   dynd/src/types/pyobject_type.cpp
                   )
 
 cython_add_module(dynd.config dynd.config_pyx True
                   dynd/include/exception_translation.hpp
-                  dynd/src/conversions.cpp
+                  dynd/src/array_conversions.cpp
+                  dynd/src/type_conversions.cpp
+                  dynd/src/type_deduction.cpp
                   ${CMAKE_CURRENT_BINARY_DIR}/dynd/src/git_version.cpp
                   )
 
 foreach(module dynd.ndt.type dynd.ndt.json dynd.nd.callable dynd.nd.functional dynd.nd.registry)
-    cython_add_module(${module} ${module}_pyx True dynd/src/conversions.cpp)
+    cython_add_module(${module} ${module}_pyx True
+                      # Additional C++ source files:
+                      dynd/src/type_conversions.cpp
+                      dynd/src/array_conversions.cpp)
 endforeach(module)
 
+
 # Run a postprocess script to work around some Cython bugs
 # that haven't been fixed in the latest release.
 postprocess_cython( postprocess.py dynd.ndt.type_postprocess dynd.ndt.type_pyx dynd.ndt.type)

dynd/cpp/types/categorical_type.pxd

@@ -1,3 +1,7 @@
+
+# Disabled until categorical_type::make takes a std::vector<T>
+
+"""
 from ..type cimport type
 from ..array cimport array
 
@@ -6,3 +10,4 @@ from ...config cimport translate_exception
 cdef extern from "dynd/types/categorical_type.hpp" namespace "dynd::ndt":
     type dynd_make_categorical_type "dynd::ndt::categorical_type::make" (array&) except +translate_exception
     type factor_categorical(array&) except +translate_exception
+"""

dynd/include/conversions.hpp renamed to dynd/include/array_conversions.hpp

@@ -4,18 +4,11 @@
 
 #include <dynd/array.hpp>
 #include <dynd/callable.hpp>
-#include <dynd/type.hpp>
 
 #include "visibility.hpp"
 
 namespace pydynd {
 
-PYDYND_API dynd::ndt::type &type_to_cpp_ref(PyObject *);
-PYDYND_API PyTypeObject *get_type_pytypeobject();
-PYDYND_API PyObject *type_from_cpp(const dynd::ndt::type &);
-PYDYND_API dynd::ndt::type dynd_ndt_as_cpp_type(PyObject *);
-PYDYND_API dynd::ndt::type dynd_ndt_cpp_type_for(PyObject *);
-
 PYDYND_API dynd::nd::array &array_to_cpp_ref(PyObject *);
 PYDYND_API PyTypeObject *get_array_pytypeobject();
 PYDYND_API PyObject *array_from_cpp(const dynd::nd::array &);

dynd/include/array_from_py.hpp

@@ -10,7 +10,6 @@
 
 #include <dynd/types/type_id.hpp>
 
-#include "array_from_py_typededuction.hpp"
 #include "visibility.hpp"
 
 #include <dynd/array.hpp>
@@ -35,24 +34,6 @@ PYDYND_API dynd::nd::array array_from_py(PyObject *obj, uint32_t access_flags,
 
 PYDYND_API dynd::ndt::type xtype_for_prefix(PyObject *obj);
 
-inline dynd::ndt::type xarray_from_pylist(PyObject *obj)
-{
-  // TODO: Add ability to specify access flags (e.g. immutable)
-  // Do a pass through all the data to deduce its type and shape
-  std::vector<intptr_t> shape;
-  dynd::ndt::type tp(dynd::void_id);
-  Py_ssize_t size = PyList_GET_SIZE(obj);
-  shape.push_back(size);
-  for (Py_ssize_t i = 0; i < size; ++i) {
-    deduce_pylist_shape_and_dtype(PyList_GET_ITEM(obj, i), shape, tp, 1);
-  }
-
-  if (tp.get_id() == dynd::void_id) {
-    tp = dynd::ndt::type(dynd::int32_id);
-  }
-
-  return dynd::ndt::make_type(shape.size(), shape.data(), tp);
-}
 
 void init_array_from_py();
 

dynd/include/array_functions.hpp

@@ -26,10 +26,10 @@
 #include "array_as_numpy.hpp"
 #include "array_as_pep3118.hpp"
 #include "array_from_py.hpp"
-#include "conversions.hpp"
+#include "array_conversions.hpp"
+#include "utility_functions.hpp"
 #include "type_functions.hpp"
 #include "types/pyobject_type.hpp"
-#include "utility_functions.hpp"
 #include "visibility.hpp"
 
 namespace pydynd {

dynd/include/kernels/apply_jit_kernel.hpp

@@ -3,9 +3,9 @@
 #include <dynd/kernels/base_kernel.hpp>
 #include <array_functions.hpp>
 #include <utility_functions.hpp>
-#include <type_functions.hpp>
 
-#include "conversions.hpp"
+#include "type_functions.hpp"
+#include "array_conversions.hpp"
 
 namespace pydynd {
 namespace nd {

dynd/include/kernels/apply_pyobject_kernel.hpp

@@ -3,6 +3,7 @@
 #include <dynd/kernels/base_kernel.hpp>
 #include <dynd/func/assignment.hpp>
 
+#include "array_conversions.hpp"
 #include "type_functions.hpp"
 #include "types/pyobject_type.hpp"
 

dynd/include/kernels/assign_from_pyobject_kernel.hpp

@@ -14,11 +14,11 @@
 #include <dynd/types/categorical_type.hpp>
 #include <dynd/types/var_dim_type.hpp>
 
-#include "array_from_py_typededuction.hpp"
 #include "array_functions.hpp"
-#include "conversions.hpp"
+#include "array_conversions.hpp"
 #include "copy_from_numpy_arrfunc.hpp"
 #include "type_functions.hpp"
+#include "type_deduction.hpp"
 #include "types/pyobject_type.hpp"
 
 using namespace dynd;

dynd/include/numpy_interop.hpp

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include "utility_functions.hpp"
+#include "type_functions.hpp"
 
 // Define this to 1 or 0 depending on whether numpy interop
 // should be compiled in.
@@ -48,6 +49,7 @@
 #include <dynd/array.hpp>
 #include <dynd/type.hpp>
 #include <dynd/types/fixed_string_type.hpp>
+#include <dynd/types/struct_type.hpp>
 
 #include <numpy/ndarrayobject.h>
 #include <numpy/ufuncobject.h>
@@ -538,6 +540,4 @@ typedef struct {
 } // namespace pydynd
 #endif // !DYND_NUMPY_INTEROP
 
-#include "type_functions.hpp"
-
 #endif // _DYND__NUMPY_INTEROP_HPP_

dynd/include/type_conversions.hpp

@@ -0,0 +1,22 @@
+//
+// Copyright (C) 2011-15 DyND Developers
+// BSD 2-Clause License, see LICENSE.txt
+//
+#pragma once
+
+#include <Python.h>
+
+#include <dynd/type.hpp>
+
+#include "visibility.hpp"
+
+namespace pydynd {
+
+PYDYND_API dynd::ndt::type &type_to_cpp_ref(PyObject *);
+PYDYND_API PyTypeObject *get_type_pytypeobject();
+PYDYND_API PyObject *type_from_cpp(const dynd::ndt::type &);
+PYDYND_API dynd::ndt::type dynd_ndt_as_cpp_type(PyObject *);
+PYDYND_API dynd::ndt::type dynd_ndt_cpp_type_for(PyObject *);
+PYDYND_API dynd::ndt::type ndt_type_from_pylist(PyObject *);
+
+} // namespace pydynd

dynd/include/array_from_py_typededuction.hpp renamed to dynd/include/type_deduction.hpp

@@ -2,19 +2,16 @@
 // Copyright (C) 2011-15 DyND Developers
 // BSD 2-Clause License, see LICENSE.txt
 //
-
-#ifndef _DYND__ARRAY_FROM_PY_TYPEDEDUCTION_HPP_
-#define _DYND__ARRAY_FROM_PY_TYPEDEDUCTION_HPP_
+#pragma once
 
 #include <Python.h>
 
-#include <dynd/array.hpp>
 #include <dynd/type.hpp>
 #include <dynd/type_promotion.hpp>
 #include <dynd/types/string_type.hpp>
 #include <dynd/types/type_id.hpp>
 
-#include "conversions.hpp"
+#include "type_conversions.hpp"
 
 namespace pydynd {
 
@@ -168,5 +165,3 @@ bool broadcast_as_scalar(const dynd::ndt::type &tp, PyObject *obj);
 void init_array_from_py_typededuction();
 
 } // namespace pydynd
-
-#endif // _DYND__ARRAY_FROM_PY_TYPEDEDUCTION_HPP_

dynd/include/type_functions.hpp

@@ -13,23 +13,22 @@
 #include <dynd/string_encodings.hpp>
 #include <dynd/type.hpp>
 
-#include "conversions.hpp"
-#include "numpy_interop.hpp"
-#include "utility_functions.hpp"
-#include "visibility.hpp"
-
-#include <dynd/shape_tools.hpp>
 #include <dynd/types/bytes_type.hpp>
 #include <dynd/types/fixed_dim_type.hpp>
 #include <dynd/types/fixed_string_type.hpp>
-//#include <dynd/types/pointer_type.hpp>
 #include <dynd/types/string_type.hpp>
 #include <dynd/types/struct_type.hpp>
 #include <dynd/types/type_type.hpp>
 
+#include "utility_functions.hpp"
+#include "type_conversions.hpp"
+#include "visibility.hpp"
+
+
 // Python's datetime C API
 #include "datetime.h"
 
+
 namespace pydynd {
 
 inline std::string _type_str(const dynd::ndt::type &d)
@@ -59,9 +58,6 @@ inline PyObject *_type_get_shape(const dynd::ndt::type &d)
   }
 }
 
-inline dynd::ndt::type
-dynd_make_fixed_dim_type(PyObject *shape, const dynd::ndt::type &element_tp);
-
 inline dynd::string_encoding_t encoding_from_pyobject(PyObject *encoding_obj)
 {
   // Default is utf-8
@@ -143,9 +139,9 @@ inline dynd::ndt::type dynd_make_fixed_string_type(intptr_t size,
   return dynd::ndt::fixed_string_type::make(size, encoding);
 }
 
-inline dynd::ndt::type dynd_make_string_type(PyObject *encoding_obj)
+inline dynd::ndt::type dynd_make_string_type(PyObject *DYND_UNUSED(encoding_obj))
 {
-  dynd::string_encoding_t encoding = encoding_from_pyobject(encoding_obj);
+  // dynd::string_encoding_t encoding = encoding_from_pyobject(encoding_obj);
 
   return dynd::ndt::make_type<dynd::ndt::string_type>();
 }
@@ -155,6 +151,18 @@ inline dynd::ndt::type dynd_make_pointer_type(const dynd::ndt::type &target_tp)
   return dynd::ndt::pointer_type::make(target_tp);
 }
 
+inline void
+pyobject_as_vector__type(PyObject *list_of_types,
+                         std::vector<dynd::ndt::type> &vector_of__types)
+{
+  Py_ssize_t size = PySequence_Size(list_of_types);
+  vector_of__types.resize(size);
+  for (Py_ssize_t i = 0; i < size; ++i) {
+    pyobject_ownref item(PySequence_GetItem(list_of_types, i));
+    vector_of__types[i] = dynd_ndt_as_cpp_type(item.get());
+  }
+}
+
 inline dynd::ndt::type dynd_make_struct_type(PyObject *field_types,
                                              PyObject *field_names)
 {