Support types mapping include float (#532)

* support float and types mapping,change eslint complexity back to 20 , add bn.js to dependency in common

* Fix bn.js dependency

* change BN to bn

* remove bn.js in dependencies

* fix extractType

Author: jiqiang90

.eslintrc.js

@@ -66,7 +66,7 @@ module.exports = {
     // "@typescript-eslint/no-param-reassign": "error",
     '@typescript-eslint/promise-function-async': ['error', {checkArrowFunctions: false}],
     // "arrow-body-style": "error",
-    complexity: ['error', 21],
+    complexity: ['error', 20],
     curly: ['error', 'multi-line'],
     'default-case': 'error',
     eqeqeq: ['error', 'always'],

packages/cli/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 All logs must start with the format: [x.y.z] - yyyy-mm-dd
 
 ## [Unreleased]
+### Added
+- Support Float type in Cli
 
 ## [0.15.0] - 2021-11-03
 ### Changed

packages/cli/src/controller/codegen-controller.test.ts

@@ -3,9 +3,9 @@
 
 import path from 'path';
 import {promisify} from 'util';
+
 import rimraf from 'rimraf';
 import {codegen} from './codegen-controller';
-import {transformTypes} from './types-mapping';
 
 jest.setTimeout(30000);
 
@@ -15,15 +15,6 @@ describe('Codegen can generate schema', () => {
     await promisify(rimraf)(path.join(__dirname, '../../test/schemaTest2/src'));
   });
 
-  it('can transform field into correct type', () => {
-    const testClassName = 'transformTest;';
-    expect(transformTypes(testClassName, 'ID')).toBe('string');
-    expect(transformTypes(testClassName, 'Int')).toBe('number');
-    expect(transformTypes(testClassName, 'BigInt')).toBe('bigint');
-    expect(transformTypes(testClassName, 'String')).toBe('string');
-    expect(transformTypes(testClassName, 'Date')).toBe('Date');
-  });
-
   it('codegen with correct schema should pass', async () => {
     const projectPath = path.join(__dirname, '../../test/schemaTest1');
     await codegen(projectPath);

packages/cli/src/controller/codegen-controller.ts

@@ -9,13 +9,15 @@ import {
   loadProjectManifest,
   getAllJsonObjects,
   setJsonObjectType,
+  getTypeByScalarName,
+  GraphQLEntityField,
+  GraphQLJsonFieldType,
+  GraphQLEntityIndex,
   getAllEnums,
 } from '@subql/common';
-import {GraphQLEntityField, GraphQLJsonFieldType, GraphQLEntityIndex} from '@subql/common/graphql/types';
 import ejs from 'ejs';
 import {upperFirst} from 'lodash';
 import rimraf from 'rimraf';
-import {transformTypes} from './types-mapping';
 
 const MODEL_TEMPLATE_PATH = path.resolve(__dirname, '../template/model.ts.ejs');
 const MODELS_INDEX_TEMPLATE_PATH = path.resolve(__dirname, '../template/models-index.ts.ejs');
@@ -135,15 +137,14 @@ export function processFields(
       injectField.indexed = indexed;
       injectField.unique = unique;
     }
-
     if ((field as GraphQLEntityField).isEnum) {
       injectField.type = field.type;
       injectField.isEnum = true;
       injectField.isJsonInterface = false;
     } else {
       switch (field.type) {
         default: {
-          injectField.type = transformTypes(className, field.type.toString());
+          injectField.type = getTypeByScalarName(field.type).tsType;
           if (!injectField.type) {
             throw new Error(
               `Schema: undefined type "${field.type.toString()}" on field "${

packages/cli/src/controller/types-mapping.ts

@@ -7,9 +7,12 @@ type goodEntity @entity {
   field1: Int!
 
   field2: String #filed2 is an optional field
+
   field3: BigInt
 
   field4: Date
 
   field5: Test
+
+  field6: Float
 }

packages/cli/test/schemaTest1/schema.graphql

@@ -1,4 +1,3 @@
 type badEntity @entity {
-
-  id: Float #give the unknown type
+  id: BigDecimal
 }

packages/cli/test/schemaTest2/badschema.graphql

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 All logs must start with the format: [x.y.z] - yyyy-mm-dd
 
 ## [Unreleased]
+### Added
+- Add Bytes type
 
 ## [0.13.0] - 2021-11-03
 ### Added

packages/common/CHANGELOG.md

@@ -21,7 +21,8 @@
     "graphql-tag": "^2.12.5",
     "js-yaml": "^4.1.0",
     "pino": "^6.13.3",
-    "reflect-metadata": "^0.1.13"
+    "reflect-metadata": "^0.1.13",
+    "sequelize": "^6.6.2"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.4",

packages/common/package.json

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import assert from 'assert';
+import {getTypeByScalarName} from '@subql/common';
 import {
   assertListType,
   getDirectiveValues,
@@ -31,7 +32,6 @@ import {
   GraphQLRelationsType,
   IndexType,
 } from './types';
-import {isFieldScalar} from './utils';
 
 export function getAllJsonObjects(_schema: GraphQLSchema | string): GraphQLObjectType[] {
   const schema = typeof _schema === 'string' ? buildSchema(_schema) : _schema;
@@ -48,6 +48,7 @@ export function getAllEnums(_schema: GraphQLSchema | string) {
     .map((node) => node);
 }
 
+// eslint-disable-next-line complexity
 export function getAllEntitiesRelations(_schema: GraphQLSchema | string): GraphQLModelsRelationsEnums {
   const schema = typeof _schema === 'string' ? buildSchema(_schema) : _schema;
   const entities = Object.values(schema.getTypeMap())
@@ -83,7 +84,8 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string): GraphQ
       const derivedFromDirectValues = getDirectiveValues(derivedFrom, field.astNode);
       const indexDirectiveVal = getDirectiveValues(indexDirective, field.astNode);
       //If is a basic scalar type
-      if (isFieldScalar(typeString)) {
+      const typeClass = getTypeByScalarName(typeString);
+      if (typeClass?.fieldScalar) {
         newModel.fields.push(packEntityField(typeString, field, false));
       }
       // If is an enum
@@ -139,7 +141,7 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string): GraphQ
       }
       // handle indexes
       if (indexDirectiveVal) {
-        if (typeString !== 'ID' && isFieldScalar(typeString)) {
+        if (typeString !== 'ID' && typeClass) {
           newModel.indexes.push({
             unique: indexDirectiveVal.unique,
             fields: [field.name],
@@ -223,7 +225,6 @@ export function setJsonObjectType(
   return graphQLJsonObject;
 }
 
-type GraphQLNonListType = GraphQLScalarType | GraphQLObjectType<unknown, unknown>; // check | GraphQLInterfaceType | GraphQLUnionType | GraphQLEnumType;
 //Get the type, ready to be convert to string
 function extractType(type: GraphQLOutputType): string {
   if (isUnionType(type)) {

packages/common/src/graphql/entities.ts

@@ -43,16 +43,18 @@ describe('utils that handle schema.graphql', () => {
     expect(() => buildSchemaFromDocumentNode(graphqlSchema)).toThrow();
   });
 
-  it('support Bytes types', () => {
+  it('support Bytes and Float types', () => {
     const graphqlSchema = gql`
       type Test @entity {
         id: ID!
         hash: Bytes
+        rate: Float
       }
     `;
     const schema = buildSchemaFromDocumentNode(graphqlSchema);
     const entities = getAllEntitiesRelations(schema);
     expect(entities.models[0].fields[1].type).toBe('Bytes');
+    expect(entities.models[0].fields[2].type).toBe('Float');
   });
 
   it('throw error for union/enum/interface type', () => {

packages/common/src/graphql/graphql.spec.ts

@@ -3,4 +3,5 @@
 
 export * from './entities';
 export * from './schema';
+export * from './types';
 export * from './builder';

packages/common/src/graphql/index.ts

@@ -8,4 +8,5 @@ export const scalas = gql`
   scalar BigDecimal
   scalar Date
   scalar Bytes
+  scalar Float
 `;

packages/common/src/graphql/schema/scalas.ts

@@ -93,4 +93,5 @@ export enum FieldScalar {
   Date = 'Date',
   Boolean = 'Boolean',
   Bytes = 'Bytes',
+  Float = 'Float',
 }

packages/common/src/graphql/types.ts

@@ -7,3 +7,4 @@ export * from './project';
 export * from './graphql';
 export * from './query';
 export {levelFilter, Logger, LoggerOption} from './logger';
+export * from './types';

packages/common/src/graphql/utils.ts

@@ -0,0 +1,33 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import assert from 'assert';
+import {DataTypes} from 'sequelize';
+type ValueOf<T> = T[keyof T];
+export type SequelizeTypes = string | ValueOf<typeof DataTypes>;
+
+export class TypeClass {
+  constructor(
+    public name: string,
+    private _hashCode: (data: unknown) => Uint8Array,
+    private _tsType?: string,
+    private _fieldScalar?: string,
+    private _sequelizeType?: SequelizeTypes
+  ) {}
+  get tsType(): string | undefined {
+    return this._tsType;
+  }
+  get fieldScalar(): string | undefined {
+    return this._fieldScalar;
+  }
+  get sequelizeType(): SequelizeTypes {
+    assert(this._sequelizeType !== undefined, `Type ${this.name} associated sequelize type is not supported`);
+    return this._sequelizeType;
+  }
+  hashCode(data: any): Uint8Array {
+    if (this._hashCode === undefined) {
+      return Buffer.from(JSON.stringify(data));
+    }
+    return this._hashCode(data);
+  }
+}

packages/common/src/index.ts

@@ -0,0 +1,16 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {getTypeByScalarName} from '@subql/common';
+
+describe('general types', () => {
+  it('can get json type', () => {
+    const typeClass = getTypeByScalarName('Json');
+    expect(typeClass.name).toBe('Json');
+  });
+
+  it('get sequelize date type', () => {
+    const typeClass = getTypeByScalarName('Date');
+    expect(typeClass.sequelizeType).toBe('timestamp');
+  });
+});

packages/common/src/types/TypeClass.ts

@@ -0,0 +1,9 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import * as supportedTypes from './supported';
+import {TypeClass} from './TypeClass';
+
+export function getTypeByScalarName(type: string): TypeClass | undefined {
+  return Object.values(supportedTypes).find(({name}) => name === type) as TypeClass;
+}

packages/common/src/types/generalTypes.spec.ts

@@ -0,0 +1,6 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './generalTypes';
+
+export * from './TypeClass';

packages/common/src/types/generalTypes.ts

@@ -0,0 +1,15 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import BN from 'bn.js';
+import {TypeClass} from '../TypeClass';
+
+export const BigInt = new TypeClass(
+  'BigInt',
+  (data: bigint | string): Uint8Array => {
+    return new BN(data.toString()).toBuffer();
+  },
+  'bigint',
+  'BigInt',
+  'numeric'
+);

packages/common/src/types/index.ts

@@ -0,0 +1,15 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {numberToU8a} from '@polkadot/util';
+import {TypeClass} from '../TypeClass';
+
+export const Boolean = new TypeClass(
+  'Boolean',
+  (data: boolean): Uint8Array => {
+    return numberToU8a(data ? 1 : 0);
+  },
+  'boolean',
+  'Boolean',
+  'boolean'
+);

packages/common/src/types/supported/BigInt.ts

@@ -0,0 +1,20 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {isHex, hexToU8a} from '@polkadot/util';
+import {DataTypes} from 'sequelize';
+import {TypeClass} from '../TypeClass';
+
+export const Bytes = new TypeClass(
+  'Bytes',
+  (data: string | Uint8Array): Uint8Array => {
+    if (data instanceof Uint8Array) return data;
+    if (isHex(data)) {
+      return hexToU8a(data);
+    }
+    throw new Error(`can not hash ${data}`);
+  },
+  'string',
+  'Bytes',
+  DataTypes.BLOB
+);

packages/common/src/types/supported/Boolean.ts

@@ -0,0 +1,17 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import assert from 'assert';
+import {numberToU8a} from '@polkadot/util';
+import {TypeClass} from '../TypeClass';
+
+export const DateObj = new TypeClass(
+  'Date',
+  (data: Date): Uint8Array => {
+    assert(data instanceof Date, `can not hash ${data}, expect instance of Date`);
+    return numberToU8a(data.getTime());
+  },
+  'Date',
+  'Date',
+  'timestamp'
+);

packages/common/src/types/supported/Bytes.ts

@@ -0,0 +1,16 @@
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {DataTypes} from 'sequelize';
+import {TypeClass} from '../TypeClass';
+
+export const Float = new TypeClass(
+  'Float',
+  (data: number): Uint8Array => {
+    //TODO, check if this is proper way to handle float
+    return Buffer.from(data.toString());
+  },
+  'number',
+  'Float',
+  DataTypes.FLOAT
+);