diff --git a/.gitignore b/.gitignore index 1f22b9c..839c358 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,5 @@ dist .yarn/build-state.yml .yarn/install-state.gz .pnp.* +package-lock.json +test/test-app-domain \ No newline at end of file diff --git a/.mocharc.json b/.mocharc.json new file mode 100644 index 0000000..c6a4697 --- /dev/null +++ b/.mocharc.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/test*.ts", + "require": "ts-node/register" +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..2087109 --- /dev/null +++ b/package.json @@ -0,0 +1,29 @@ +{ + "name": "oak-db", + "version": "1.0.0", + "description": "oak-db", + "main": "src/index.ts", + "scripts": { + "test": "mocha", + "make:test:domain": "ts-node script/makeTestDomain.ts" + }, + "dependencies": { + "lodash": "^4.17.21", + "mysql": "^2.18.1", + "mysql2": "^2.3.3", + "oak-domain": "file:../oak-domain", + "uuid": "^8.3.2" + }, + "author": "XuChang", + "license": "ISC", + "devDependencies": { + "@types/lodash": "^4.14.182", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.42", + "@types/uuid": "^8.3.4", + "mocha": "^10.0.0", + "oak-general-business": "file:../oak-general-business", + "ts-node": "^10.8.1", + "typescript": "^4.7.3" + } +} diff --git a/script/makeTestDomain.ts b/script/makeTestDomain.ts new file mode 100644 index 0000000..23c453f --- /dev/null +++ b/script/makeTestDomain.ts @@ -0,0 +1,8 @@ +import { + buildSchema, + analyzeEntities, +} from 'oak-domain/src/compiler/schemalBuilder'; + +analyzeEntities(`${process.cwd()}/node_modules/oak-general-business/src/entities`); +analyzeEntities(`${process.cwd()}/test/entities`); +buildSchema(`${process.cwd()}/test/test-app-domain`); \ No newline at end of file diff --git a/src/MySQL/connector.ts b/src/MySQL/connector.ts new file mode 100644 index 0000000..ec76d77 --- /dev/null +++ b/src/MySQL/connector.ts @@ -0,0 +1,134 @@ +import mysql from 'mysql2'; +import { v4 } from 'uuid'; +import { TxnOption } from 'oak-domain/lib/types'; +import { MySQLConfiguration } from './types/Configuration'; +import assert from 'assert'; + +export class MySqlConnector { + pool?: mysql.Pool; + configuration: MySQLConfiguration; + txnDict: Record; + + constructor(configuration: MySQLConfiguration) { + this.configuration = configuration; + this.txnDict = {}; + } + + connect() { + this.pool = mysql.createPool(this.configuration); + } + + disconnect() { + this.pool!.end(); + } + + startTransaction(option?: TxnOption): Promise { + return new Promise( + (resolve, reject) => { + this.pool!.getConnection((err, connection) => { + if (err) { + return reject(err); + } + const { isolationLevel } = option || {}; + const startTxn = () => { + let sql = 'START TRANSACTION;'; + connection.query(sql, (err2: Error) => { + if (err2) { + connection.release(); + return reject(err2); + } + + const id = v4(); + Object.assign(this.txnDict, { + [id]: connection, + }); + + resolve(id); + }); + } + if (isolationLevel) { + connection.query(`SET TRANSACTION ISOLATION LEVEL ${isolationLevel};`, (err2: Error) => { + if (err2) { + connection.release(); + return reject(err2); + } + startTxn(); + }); + } + else { + startTxn(); + } + }) + } + ); + } + + async exec(sql: string, txn?: string): Promise { + if (txn) { + const connection = this.txnDict[txn]; + assert(connection); + + return new Promise( + (resolve, reject) => { + connection.query(sql, (err, result) => { + if (err) { + console.error(`sql exec err: ${sql}`, err); + return reject(err); + } + + resolve(result); + }); + } + ); + } + else { + return new Promise( + (resolve, reject) => { + // if (process.env.DEBUG) { + // console.log(sql); + //} + this.pool!.query(sql, (err, result) => { + if (err) { + console.error(`sql exec err: ${sql}`, err); + return reject(err); + } + + resolve(result); + }) + } + ); + } + } + + commitTransaction(txn: string): Promise { + const connection = this.txnDict[txn]; + assert(connection); + return new Promise( + (resolve, reject) => { + connection.query('COMMIT;', (err) => { + if (err) { + return reject(err); + } + connection.release(); + resolve(); + }); + } + ); + } + + rollbackTransaction(txn: string): Promise { + const connection = this.txnDict[txn]; + assert(connection); + return new Promise( + (resolve, reject) => { + connection.query('ROLLBACK;', (err: Error) => { + if (err) { + return reject(err); + } + connection.release(); + resolve(); + }); + } + ); + } +} \ No newline at end of file diff --git a/src/MySQL/store.ts b/src/MySQL/store.ts new file mode 100644 index 0000000..968a549 --- /dev/null +++ b/src/MySQL/store.ts @@ -0,0 +1,52 @@ +import { EntityDict, Context, DeduceCreateSingleOperation, DeduceRemoveOperation, DeduceUpdateOperation, OperateParams, OperationResult, SelectionResult, TxnOption, SelectRowShape, StorageSchema } from 'oak-domain/lib/types'; +import { CascadeStore } from 'oak-domain/lib/store/CascadeStore'; +import { MySQLConfiguration } from './types/Configuration'; +import { MySqlConnector } from './connector'; +import { translateCreateDatabase } from './Translator'; + +export class MysqlStore> extends CascadeStore { + connector: MySqlConnector; + constructor(storageSchema: StorageSchema, configuration: MySQLConfiguration) { + super(storageSchema); + this.connector = new MySqlConnector(configuration); + } + protected supportManyToOneJoin(): boolean { + return true; + } + protected selectAbjointRow(entity: T, Selection: S, context: Cxt, params?: OperateParams): Promise[]> { + throw new Error('Method not implemented.'); + } + protected updateAbjointRow(entity: T, operation: DeduceCreateSingleOperation | DeduceUpdateOperation | DeduceRemoveOperation, context: Cxt, params?: OperateParams): Promise { + throw new Error('Method not implemented.'); + } + operate(entity: T, operation: ED[T]['Operation'], context: Cxt, params?: OperateParams): Promise> { + throw new Error('Method not implemented.'); + } + select(entity: T, selection: S, context: Cxt, params?: Object): Promise> { + throw new Error('Method not implemented.'); + } + count(entity: T, selection: Omit, context: Cxt, params?: Object): Promise { + throw new Error('Method not implemented.'); + } + begin(option?: TxnOption): Promise { + throw new Error('Method not implemented.'); + } + commit(txnId: string): Promise { + throw new Error('Method not implemented.'); + } + rollback(txnId: string): Promise { + throw new Error('Method not implemented.'); + } + connect() { + this.connector.connect(); + } + disconnect() { + this.connector.disconnect(); + } + async initialize(dropIfExists?: boolean) { + const sql = translateCreateDatabase(this.connector.configuration.database, this.connector.configuration.charset, dropIfExists); + for (const stmt of sql) { + await this.connector.exec(stmt); + } + } +} \ No newline at end of file diff --git a/src/MySQL/translator.ts b/src/MySQL/translator.ts new file mode 100644 index 0000000..b1b5eab --- /dev/null +++ b/src/MySQL/translator.ts @@ -0,0 +1,589 @@ +import assert from 'assert'; +import { assign } from 'lodash'; +import { EntityDict, Geo, Q_FullTextValue, RefOrExpression, Ref, StorageSchema, Index } from "oak-domain/lib/types"; +import { DataType, DataTypeParams } from "oak-domain/lib/types/schema/DataTypes"; +import { SelectParams, SqlTranslator } from "../sqlTranslator"; + +const GeoTypes = [ + { + type: 'point', + name: "Point" + }, + { + type: 'path', + name: "LineString", + element: 'point', + }, + { + name: "MultiLineString", + element: "path", + multiple: true, + }, + { + type: 'polygon', + name: "Polygon", + element: "path" + }, + { + name: "MultiPoint", + element: "point", + multiple: true, + }, + { + name: "MultiPolygon", + element: "polygon", + multiple: true, + } +]; + +function transformGeoData(data: Geo): string { + if (data instanceof Array) { + const element = data[0]; + if (element instanceof Array) { + return ` GeometryCollection(${data.map( + ele => transformGeoData(ele) + ).join(',')})` + } + else { + const geoType = GeoTypes.find( + ele => ele.type === element.type + ); + if (!geoType) { + throw new Error(`${element.type} is not supported in MySQL`); + } + const multiGeoType = GeoTypes.find( + ele => ele.element === geoType.type && ele.multiple + ); + return ` ${multiGeoType!.name}(${data.map( + ele => transformGeoData(ele) + ).join(',')})`; + } + } + else { + const { type, coordinate } = data; + const geoType = GeoTypes.find( + ele => ele.type === type + ); + if (!geoType) { + throw new Error(`${data.type} is not supported in MySQL`); + } + const { element, name } = geoType; + if (!element) { + // Point + return ` ${name}(${coordinate.join(',')})`; + } + // Polygon or Linestring + return ` ${name}(${coordinate.map( + (ele) => transformGeoData({ + type: element as any, + coordinate: ele as any, + }) + )})`; + } +} + +type IndexHint = { + $force?: string; + $ignore?: string; +} & { + [k: string]: IndexHint; +} + +interface MySqlSelectParams extends SelectParams { + indexHint?: IndexHint; +} + +export class MySqlTranslator extends SqlTranslator { + private modifySchema() { + for (const entity in this.schema) { + const { attributes, indexes } = this.schema[entity]; + const geoIndexes: Index[] = []; + for (const attr in attributes) { + if (attributes[attr].type === 'geometry') { + const geoIndex = indexes?.find( + (idx) => idx.config?.type === 'spatial' && idx.attributes.find( + (attrDef) => attrDef.name === attr + ) + ); + if (!geoIndex) { + geoIndexes.push({ + name: `${entity}_geo_${attr}`, + attributes: [{ + name: attr, + }], + config: { + type: 'spatial', + } + }); + } + } + } + + if (geoIndexes.length > 0) { + if (indexes) { + indexes.push(...geoIndexes); + } + else { + assign(this.schema[entity], { + indexes: geoIndexes, + }); + } + } + } + } + + constructor(schema: StorageSchema) { + super(schema); + // MySQL为geometry属性默认创建索引 + this.modifySchema(); + } + static supportedDataTypes: DataType[] = [ + // numeric types + "bit", + "int", + "integer", // synonym for int + "tinyint", + "smallint", + "mediumint", + "bigint", + "float", + "double", + "double precision", // synonym for double + "real", // synonym for double + "decimal", + "dec", // synonym for decimal + "numeric", // synonym for decimal + "fixed", // synonym for decimal + "bool", // synonym for tinyint + "boolean", // synonym for tinyint + // date and time types + "date", + "datetime", + "timestamp", + "time", + "year", + // string types + "char", + "nchar", // synonym for national char + "national char", + "varchar", + "nvarchar", // synonym for national varchar + "national varchar", + "blob", + "text", + "tinyblob", + "tinytext", + "mediumblob", + "mediumtext", + "longblob", + "longtext", + "enum", + "set", + "binary", + "varbinary", + // json data type + "json", + // spatial data types + "geometry", + "point", + "linestring", + "polygon", + "multipoint", + "multilinestring", + "multipolygon", + "geometrycollection" + ]; + + static spatialTypes: DataType[] = [ + "geometry", + "point", + "linestring", + "polygon", + "multipoint", + "multilinestring", + "multipolygon", + "geometrycollection" + ]; + + static withLengthDataTypes: DataType[] = [ + "char", + "varchar", + "nvarchar", + "binary", + "varbinary" + ]; + + static withPrecisionDataTypes: DataType[] = [ + "decimal", + "dec", + "numeric", + "fixed", + "float", + "double", + "double precision", + "real", + "time", + "datetime", + "timestamp" + ]; + + static withScaleDataTypes: DataType[] = [ + "decimal", + "dec", + "numeric", + "fixed", + "float", + "double", + "double precision", + "real" + ]; + + static unsignedAndZerofillTypes: DataType[] = [ + "int", + "integer", + "smallint", + "tinyint", + "mediumint", + "bigint", + "decimal", + "dec", + "numeric", + "fixed", + "float", + "double", + "double precision", + "real" + ]; + + static withWidthDataTypes: DataType[] = [ + 'int', + ] + + static dataTypeDefaults = { + "varchar": { length: 255 }, + "nvarchar": { length: 255 }, + "national varchar": { length: 255 }, + "char": { length: 1 }, + "binary": { length: 1 }, + "varbinary": { length: 255 }, + "decimal": { precision: 10, scale: 0 }, + "dec": { precision: 10, scale: 0 }, + "numeric": { precision: 10, scale: 0 }, + "fixed": { precision: 10, scale: 0 }, + "float": { precision: 12 }, + "double": { precision: 22 }, + "time": { precision: 0 }, + "datetime": { precision: 0 }, + "timestamp": { precision: 0 }, + "bit": { width: 1 }, + "int": { width: 11 }, + "integer": { width: 11 }, + "tinyint": { width: 4 }, + "smallint": { width: 6 }, + "mediumint": { width: 9 }, + "bigint": { width: 20 } + }; + + maxAliasLength = 63; + private populateDataTypeDef(type: DataType | Ref, params?: DataTypeParams): string{ + if (MySqlTranslator.withLengthDataTypes.includes(type as DataType)) { + if (params) { + const { length } = params; + return `${type}(${length}) `; + } + else { + const { length } = (MySqlTranslator.dataTypeDefaults as any)[type]; + return `${type}(${length}) `; + } + } + + if (MySqlTranslator.withPrecisionDataTypes.includes(type as DataType)) { + if (params) { + const { precision, scale } = params; + if (typeof scale === 'number') { + return `${type}(${precision}, ${scale}) `; + } + return `${type}(${precision})`; + } + else { + const { precision, scale } = (MySqlTranslator.dataTypeDefaults as any)[type]; + if (typeof scale === 'number') { + return `${type}(${precision}, ${scale}) `; + } + return `${type}(${precision})`; + } + } + + if (MySqlTranslator.withWidthDataTypes.includes(type as DataType)) { + assert(type === 'int'); + const { width } = params!; + switch(width!) { + case 1: { + return 'tinyint'; + } + case 2: { + return 'smallint'; + } + case 3: { + return 'mediumint'; + } + case 4: { + return 'int'; + } + default: { + return 'bigint'; + } + } + } + + if (['date'].includes(type)) { + return 'bigint '; // 因为历史原因,date类型用bigint存,Date.now() + } + if (['object', 'array'].includes(type)) { + return 'text '; + } + if (['image', 'function'].includes(type)) { + return 'text '; + } + if (type === 'ref') { + return 'char(36)'; + } + + return `${type} `; + } + + protected translateAttrProjection(dataType: DataType, alias: string, attr: string): string { + switch(dataType) { + case 'geometry': { + return ` st_astext(\`${alias}\`.\`${attr}\`)`; + } + default:{ + return ` \`${alias}\`.\`${attr}\``; + } + } + } + + protected translateAttrValue(dataType: DataType, value: any): string { + if (value === null) { + return 'null'; + } + switch (dataType) { + case 'geometry': { + return transformGeoData(value); + } + case 'date': { + if (value instanceof Date) { + return `${value.valueOf()}`; + } + else if (typeof value === 'number') { + return `${value}`; + } + return value as string; + } + case 'object': + case 'array': { + return this.escapeStringValue(JSON.stringify(value)); + } + /* case 'function': { + return `'${Buffer.from(value.toString()).toString('base64')}'`; + } */ + default: { + if (typeof value === 'string') { + return this.escapeStringValue(value); + } + return value as string; + } + } + } + protected translateFullTextSearch(value: Q_FullTextValue, entity: T, alias: string): string { + const { $search } = value; + const { indexes } = this.schema[entity]; + + const ftIndex = indexes && indexes.find( + (ele) => { + const { config } = ele; + return config && config.type === 'fulltext'; + } + ); + assert(ftIndex); + const { attributes } = ftIndex; + const columns2 = attributes.map( + ({ name }) => `${alias}.${name as string}` + ); + return ` match(${columns2.join(',')}) against ('${$search}' in natural language mode)`; + } + translateCreateEntity(entity: T, options?: { replace?: boolean; }): string { + const replace = options?.replace; + const { schema } = this; + const entityDef = schema[entity]; + const { storageName, attributes, indexes, view } = entityDef; + + // todo view暂还不支持 + const entityType = view ? 'view' : 'table'; + let sql = !replace ? `create ${entityType} if not exists ` : `create ${entityType} `; + if (storageName) { + sql += `\`${storageName}\` `; + } + else { + sql += `\`${entity as string}\` `; + } + + if (view) { + throw new Error(' view unsupported yet'); + } + else { + sql += '('; + // 翻译所有的属性 + Object.keys(attributes).forEach( + (attr, idx) => { + const attrDef = attributes[attr]; + const { + type, + params, + default: defaultValue, + unique, + notNull, + } = attrDef; + if (type === 'ref') { + return; + } + sql += `\`${attr}\` ` + sql += this.populateDataTypeDef(type, params) as string; + + if (notNull) { + sql += ' not null '; + } + if (unique) { + sql += ' unique '; + } + if (defaultValue !== undefined) { + sql += ` default ${this.translateAttrValue(type, defaultValue)}`; + } + if (attr === 'id') { + sql += ' primary key' + } + if (idx < Object.keys(attributes).length - 1) { + sql += ',\n'; + } + } + ); + + // 翻译索引信息 + if (indexes) { + sql += ',\n'; + indexes.forEach( + ({ name, attributes, config }, idx) => { + const { unique, type, parser } = config || {}; + if (unique) { + sql += ' unique '; + } + else if (type === 'fulltext') { + sql += ' fulltext '; + } + else if (type === 'spatial') { + sql += ' spatial '; + } + sql += `index ${name} `; + if (type === 'hash') { + sql += ` using hash `; + } + sql += '('; + + let includeDeleteAt = false; + attributes.forEach( + ({ name, size, direction }, idx2) => { + sql += `\`${name as string}\``; + if (size) { + sql += ` (${size})`; + } + if (direction) { + sql += ` ${direction}`; + } + if (idx2 < attributes.length - 1) { + sql += ',' + } + if (name === '$$deleteAt$$') { + includeDeleteAt = true; + } + } + ); + if (!includeDeleteAt && !type) { + sql += ', $$deleteAt$$'; + } + sql += ')'; + if (parser) { + sql += ` with parser ${parser}`; + } + if (idx < indexes.length - 1) { + sql += ',\n'; + } + } + ); + } + } + + + sql += ')'; + + return sql; + } + protected translateExpression(alias: string, expression: RefOrExpression, refDict: Record): string { + throw new Error("Method not implemented."); + } + protected populateSelectStmt(projectionText: string, fromText: string, aliasDict: Record, filterText: string, sorterText?: string, indexFrom?: number, count?: number, params?: MySqlSelectParams): string { + // todo using index + let sql = `select ${projectionText} from ${fromText}`; + if (filterText) { + sql += ` where ${filterText}`; + } + if (sorterText) { + sql += ` order by ${sorterText}`; + } + if (typeof indexFrom === 'number') { + assert (typeof count === 'number'); + sql += ` limit ${indexFrom}, ${count}`; + } + if (params?.forUpdate) { + sql += ' for update'; + } + sql += ';'; + + return sql; + } + protected populateUpdateStmt(updateText: string, fromText: string, aliasDict: Record, filterText: string, sorterText?: string, indexFrom?: number, count?: number, params?: MySqlSelectParams): string { + // todo using index + const alias = aliasDict['./']; + let sql = `update ${fromText} set ${updateText}, \`${alias}\`.\`$$updateAt$$\` = ${Date.now()}`; + if (filterText) { + sql += ` where ${filterText}`; + } + if (sorterText) { + sql += ` order by ${sorterText}`; + } + if (typeof indexFrom === 'number') { + assert (typeof count === 'number'); + sql += ` limit ${indexFrom}, ${count}`; + } + sql += ';'; + + return sql; + } + protected populateRemoveStmt(removeText: string, fromText: string, aliasDict: Record, filterText: string, sorterText?: string, indexFrom?: number, count?: number, params?: MySqlSelectParams): string { + // todo using index + const alias = aliasDict['./']; + let sql = `update ${fromText} set \`${alias}\`.\`$$removeAt$$\` = ${Date.now()}`; + if (filterText) { + sql += ` where ${filterText}`; + } + if (sorterText) { + sql += ` order by ${sorterText}`; + } + if (typeof indexFrom === 'number') { + assert (typeof count === 'number'); + sql += ` limit ${indexFrom}, ${count}`; + } + sql += ';'; + + return sql; + } +} \ No newline at end of file diff --git a/src/MySQL/types/Configuration.ts b/src/MySQL/types/Configuration.ts new file mode 100644 index 0000000..e58e8f6 --- /dev/null +++ b/src/MySQL/types/Configuration.ts @@ -0,0 +1,12 @@ +export type MySQLConfiguration = { + host: string; + user: string; + password: string; + database: string; + charset: 'utf8mb4_general_ci'; + connectionLimit: number; +} + +export type Configuration = { + mysql: MySQLConfiguration; +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/sqlTranslator.ts b/src/sqlTranslator.ts new file mode 100644 index 0000000..2e8c3b1 --- /dev/null +++ b/src/sqlTranslator.ts @@ -0,0 +1,831 @@ +import assert from 'assert'; +import { assign, cloneDeep, keys, set } from 'lodash'; +import { Attribute, DeduceCreateOperationData, DeduceSorterAttr, DeduceSorterItem, EntityDict, Expression, EXPRESSION_PREFIX, Index, Q_FullTextValue, RefOrExpression, StorageSchema } from "oak-domain/lib/types"; +import { DataType } from "oak-domain/lib/types/schema/DataTypes"; +import { judgeRelation } from 'oak-domain/lib/store/relation'; + +export type SelectParams = { + forUpdate?: boolean; +}; + +export abstract class SqlTranslator { + readonly schema: StorageSchema; + constructor(schema: StorageSchema) { + this.schema = this.makeFullSchema(schema); + } + + private makeFullSchema(schema2: StorageSchema) { + const schema = cloneDeep(schema2); + for (const entity in schema) { + const { attributes, indexes } = schema[entity]; + // 增加默认的属性 + assign(attributes, { + id: { + type: 'char', + params: { + length: 36, + }, + } as Attribute, + $$createAt$$: { + type: 'date', + notNull: true, + } as Attribute, + $$updateAt$$: { + type: 'date', + notNull: true, + } as Attribute, + $$removeAt$$: { + type: 'date', + } as Attribute, + $$triggerData$$: { + type: 'object', + } as Attribute, + $$triggerTimestamp$$: { + type: 'date', + } as Attribute, + }); + + // 增加默认的索引 + const intrinsticIndexes: Index[] = [ + { + name: `${entity}_create_at`, + attributes: [{ + name: '$$createAt$$', + }] + }, { + name: `${entity}_update_at`, + attributes: [{ + name: '$$updateAt$$', + }], + }, { + name: `${entity}_trigger_ts`, + attributes: [{ + name: '$$triggerTimestamp$$', + }], + } + ]; + + // 增加外键上的索引 + for (const attr in attributes) { + if (attributes[attr].type === 'ref') { + if (!(indexes?.find( + ele => ele.attributes[0].name === attr + ))) { + intrinsticIndexes.push({ + name: `${entity}_fk_${attr}`, + attributes: [{ + name: attr, + }] + }); + } + } + + if (attr === 'entity' && attributes[attr].type === 'varchar') { + const entityIdDef = attributes.entityId; + if (entityIdDef?.type === 'varchar') { + if (!(indexes?.find( + ele => ele.attributes[0].name === 'entity' && ele.attributes[1]?.name === 'entityId' + ))) { + intrinsticIndexes.push({ + name: `${entity}_fk_entity_entityId`, + attributes: [{ + name: 'entity', + }, { + name: 'entityId', + }] + }); + } + } + } + } + + if (indexes) { + indexes.push(...intrinsticIndexes); + } + else { + assign(schema[entity], { + indexes: intrinsticIndexes, + }); + } + } + + return schema; + } + + + protected abstract translateAttrProjection(dataType: DataType, alias: string, attr: string): string; + + protected abstract translateAttrValue(dataType: DataType, value: any): string; + + protected abstract translateFullTextSearch(value: Q_FullTextValue, entity: T, alias: string): string; + + abstract translateCreateEntity(entity: T, option: { replace?: boolean }): string; + + protected abstract populateSelectStmt( + projectionText: string, + fromText: string, + aliasDict: Record, + filterText: string, + sorterText?: string, + indexFrom?: number, + count?: number, + params?: SelectParams): string; + + protected abstract populateUpdateStmt( + updateText: string, + fromText: string, + aliasDict: Record, + filterText: string, + sorterText?: string, + indexFrom?: number, + count?: number, + params?: any): string; + + protected abstract populateRemoveStmt( + removeText: string, + fromText: string, + aliasDict: Record, + filterText: string, + sorterText?: string, + indexFrom?: number, + count?: number, + params?: any): string; + + protected abstract translateExpression(alias: string, expression: RefOrExpression, refDict: Record): string; + + private getStorageName(entity: T) { + const { storageName } = this.schema[entity]; + return (storageName || entity) as string; + } + + translateInsert(entity: T, data: DeduceCreateOperationData[]): string { + const { schema } = this; + const { attributes, storageName = entity } = schema[entity]; + + let sql = `insert into \`${storageName as string}\`(`; + + const attrs = Object.keys(data[0]).filter( + ele => attributes.hasOwnProperty(ele) && attributes[ele].type !== 'ref' + ); + attrs.forEach( + (attr, idx) => { + sql += ` \`${attr}\``; + if (idx < attrs.length - 1) { + sql += ','; + } + } + ); + + sql += ' `$$createAt$$, $$updateAt$$) values '; + + const now = Date.now(); + data.forEach( + (d, dataIndex) => { + sql += '('; + attrs.forEach( + (attr, attrIdx) => { + const attrDef = attributes[attr]; + const { type: dataType } = attrDef; + const value = this.translateAttrValue(dataType as DataType, d[attr]); + sql += value; + if (attrIdx < attrs.length - 1) { + sql += ','; + } + } + ); + sql += `, ${now}, ${now})`; + if (dataIndex < data.length - 1) { + sql += ','; + } + } + ); + + return sql; + } + + /** + * analyze the join relations in projection/query/sort + * 所有的层次关系都当成left join处理,如果有内表为空的情况,请手动处理 + * { + * b: { + * name: { + * $exists: false, + * } +* } + * } + * 这样的query会把内表为空的行也返回 + * @param param0 + */ + private analyzeJoin(entity: T, { projection, filter, sorter, isStat }: { + projection?: ED[T]['Selection']['data']; + filter?: ED[T]['Selection']['filter']; + sorter?: ED[T]['Selection']['sorter']; + isStat?: true; + }): { + aliasDict: Record; + projectionRefAlias: Record; + filterRefAlias: Record; + from: string; + extraWhere: string; + } { + const { schema } = this; + let count = 1; + const projectionRefAlias: Record = {}; + const filterRefAlias: Record = {}; + let extraWhere = ''; + + const alias = `${entity as string}_${count++}`; + let from = ` \`${this.getStorageName(entity)}\` \`${alias}\` `; + const aliasDict: Record = { + './': alias, + }; + + const analyzeFilterNode = ({ node, path, entityName, alias }: { + node: ED[E]['Selection']['filter']; + path: string; + entityName: E; + alias: string, + }): void => { + Object.keys(node!).forEach( + (op) => { + if (['$and', '$or'].includes(op)) { + (node![op] as ED[E]['Selection']['filter'][]).forEach( + (subNode) => analyzeFilterNode({ + node: subNode, + path, + entityName, + alias, + }) + ); + } + else { + const rel = judgeRelation(this.schema, entityName, op); + if (typeof rel === 'string') { + let alias2: string; + const pathAttr = `${path}${op}/`; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${rel}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(rel)}\` \`${alias2}\` on \`${alias}\`.\`${op}Id\` = \`${alias2}\`.\`id\``; + } + else { + alias2 = aliasDict[pathAttr]; + } + analyzeFilterNode({ + node: node![op], + path: pathAttr, + entityName: rel, + alias: alias2, + }); + } + else if (rel === 2) { + let alias2: string; + const pathAttr = `${path}${op}/`; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${op}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(op)}\` \`${alias2}\` on \`${alias}\`.\`entityId\` = \`${alias2}\`.\`id\``; + extraWhere += `\`${alias}\`.\`entity\` = '${op}'`; + } + else { + alias2 = aliasDict[pathAttr]; + } + analyzeFilterNode({ + node: node![op], + path: pathAttr, + entityName: rel, + alias: alias2, + }); + } + else { + // 不支持一对多 + assert(rel === 0 || rel === 1); + } + } + } + ); + if (node!['#id']) { + assert(!filterRefAlias[node!['#id']]); + assign(filterRefAlias, { + [node!['#id']]: alias, + }); + } + }; + if (filter) { + analyzeFilterNode({ + node: filter, + path: './', + entityName: entity, + alias, + }); + } + + const analyzeSortNode = ({ node, path, entityName, alias }: { + node: DeduceSorterAttr; + path: string; + entityName: E; + alias: string; + }): void => { + const attr = keys(node)[0]; + + const rel = judgeRelation(this.schema, entityName, attr); + if (typeof rel === 'string') { + const pathAttr = `${path}${attr}/`; + let alias2: string; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${rel}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(rel)}\` \`${alias2}\` on \`${alias}\`.\`${attr}Id\` = \`${alias2}\`.\`id\``; + } + else { + alias2 = aliasDict[pathAttr]; + } + analyzeSortNode({ + node: node[attr] as any, + path: pathAttr, + entityName: rel, + alias: alias2, + }); + } + else if (rel === 2) { + const pathAttr = `${path}${attr}/`; + let alias2: string; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${attr}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(attr)}\` \`${alias2}\` on \`${alias}\`.\`entityId\` = \`${alias2}\`.\`id\``; + extraWhere += `\`${alias}\`.\`entity\` = '${attr}'`; + } + else { + alias2 = aliasDict[pathAttr]; + } + analyzeSortNode({ + node: node[attr] as any, + path: pathAttr, + entityName: attr, + alias: alias2, + }); + } + else { + assert(rel === 0 || rel === 1); + } + }; + if (sorter) { + sorter.forEach( + (sortNode) => { + analyzeSortNode({ + node: sortNode.$attr, + path: './', + entityName: entity, + alias, + }); + } + ); + } + + const analyzeProjectionNode = ({ node, path, entityName, alias }: { + node: ED[E]['Selection']['data']; + path: string; + entityName: E; + alias: string; + }): void => { + const { attributes } = schema[entityName]; + + if (!isStat && attributes.hasOwnProperty('id') && !node.id) { + assign(node, { + id: 1, + }); + } + Object.keys(node).forEach( + (attr) => { + const rel = judgeRelation(this.schema, entityName, attr); + if (typeof rel === 'string') { + const pathAttr = `${path}${attr}/`; + + let alias2: string; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${rel}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(rel)}\` \`${alias2}\` on \`${alias}\`.\`${attr}Id\` = \`${alias2}\`.\`id\``; + } + else { + alias2 = aliasDict[pathAttr]; + } + + analyzeProjectionNode({ + node: node[attr], + path: pathAttr, + entityName: rel, + alias: alias2, + }); + } + else if (rel === 2) { + const pathAttr = `${path}${attr}/`; + + let alias2: string; + if (!aliasDict.hasOwnProperty(pathAttr)) { + alias2 = `${attr}_${count++}`; + assign(aliasDict, { + [pathAttr]: alias2, + }); + from += ` left join \`${this.getStorageName(attr)}\` \`${alias2}\` on \`${alias}\`.\`entityId\` = \`${alias2}\`.\`id\``; + } + else { + alias2 = aliasDict[pathAttr]; + } + + analyzeProjectionNode({ + node: node[attr], + path: pathAttr, + entityName: attr, + alias: alias2, + }); + extraWhere += `\`${alias}\`.\`entity\` = '${attr}'`; + } + } + ); + if (node['#id']) { + assert(!projectionRefAlias[node['#id']]); + assign(projectionRefAlias, { + [node['#id']]: alias, + }); + } + }; + + if (projection) { + analyzeProjectionNode({ node: projection, path: './', entityName: entity, alias }); + } + + return { + aliasDict, + from, + projectionRefAlias, + filterRefAlias, + extraWhere, + }; + } + + private translateComparison(attr: string, value: any, type: DataType): string { + const SQL_OP: { + [op: string]: string, + } = { + $gt: '>', + $lt: '<', + $gte: '>=', + $lte: '<=', + $eq: '=', + $ne: '<>', + }; + + if (Object.keys(SQL_OP).includes(attr)) { + return ` ${SQL_OP[attr]} ${this.translateAttrValue(type, value)}`; + } + + switch (attr) { + case '$startsWith': { + return ` like '${value}%'`; + } + case '$endsWith': { + return ` like '%${value}'`; + } + case '$includes': { + return ` like '%${value}%'`; + } + default: { + throw new Error(`unrecoganized comparison operator ${attr}`); + } + } + } + + private translateElement(attr: string, value: boolean): string { + assert(attr === '$exists'); // only support one operator now + if (value) { + return ' is not null'; + } + return ' is null'; + } + + private translateEvaluation(attr: string, value: any, entity: T, alias: string, type: DataType): string { + switch (attr) { + case '$text': { + // fulltext search + return this.translateFullTextSearch(value, entity, alias); + } + case '$in': + case '$nin': { + const IN_OP = { + $in: 'in', + $nin: 'not in', + }; + if (value instanceof Array) { + const values = value.map( + (v) => { + if (['varchar', 'char', 'text', 'nvarchar'].includes(type as string)) { + return `'${v}'`; + } + else { + return `${v}`; + } + } + ); + if (values.length > 0) { + return ` ${IN_OP[attr]}(${values.join(',')})`; + } + else { + if (attr === '$in') { + return ' in (null)'; + } + else { + return ' is not null'; + } + } + } + else { + // sub query + return ` ${IN_OP[attr]}(${this.translateSelect(value.$entity, value)})`; + } + } + default: { + assert('$between' === attr); + const values = value.map( + (v: string | number) => { + if (['varchar', 'char', 'text', 'nvarchar'].includes(type as string)) { + return `'${v}'`; + } + else { + return `${v}`; + } + } + ); + return ` between ${values[0]} and ${values[1]}`; + } + } + } + + private translateFilter( + entity: T, + aliasDict: Record, + filterRefAlias: Record, + filter?: ED[T]['Selection']['filter'], + extraWhere?: string): string { + const { schema } = this; + + const translateInner = (entity2: E, path: string, filter2?: ED[E]['Selection']['filter'], type?: DataType): string => { + const alias = aliasDict[path]; + const { attributes } = schema[entity2]; + let whereText = ''; + if (filter2) { + Object.keys(filter2).forEach( + (attr, idx) => { + whereText + '('; + if (['$and', '$or', '$xor', '$not'].includes(attr)) { + let result = ''; + switch (attr) { + case '$and': + case '$or': + case '$xor': { + const logicQueries = filter2[attr]; + logicQueries.forEach( + (logicQuery: ED[E]['Selection']['filter'], index: number) => { + const sql = translateInner(entity2, path, logicQuery); + if (sql) { + whereText += ` (${sql})`; + if (index < logicQueries.length - 1) { + whereText += ` ${attr.slice(1)}`; + } + } + } + ); + break; + } + default: { + assert(attr === '$not'); + const logicQuery = filter2[attr]; + const sql = translateInner(entity2, path, logicQuery); + if (sql) { + whereText += ` not (${translateInner(entity2, path, logicQuery)})`; + break; + } + } + } + } + else if (attr.toLowerCase().startsWith(EXPRESSION_PREFIX)) { + // expression + whereText += ` (${this.translateExpression(alias, filter2[attr], filterRefAlias)}) as ${attr}`; + } + else if (['$gt', '$gte', '$lt', '$lte', '$eq', '$ne', '$startsWith', '$endsWith', '$includes'].includes(attr)) { + whereText += this.translateComparison(attr, filter2[attr], type!); + } + else if (['$exists'].includes(attr)) { + whereText += this.translateElement(attr, filter2[attr]); + } + else if (['$text', '$in', '$nin', '$between'].includes(attr)) { + whereText += this.translateEvaluation(attr, filter2[attr], entity2, alias, type!); + } + else { + assert(attributes.hasOwnProperty(attr)); + const { type: type2, ref } = attributes[attr]; + if (type2 === 'ref') { + whereText += ` ${translateInner(ref!, `${path}${attr}/`, filter2[attr])}`; + } + else if (typeof filter2[attr] === 'object' && Object.keys(filter2[attr])[0] && Object.keys(filter2[attr])[0].startsWith('$')) { + whereText += ` \`${alias}\`.\`${attr}\` ${translateInner(entity2, path, filter2[attr], type2)}` + } + else { + whereText += ` \`${alias}\`.\`${attr}\` = ${this.translateAttrValue(type2, filter2[attr])}`; + } + } + + whereText + ')'; + if (idx < Object.keys(filter2).length - 1) { + whereText += ' and' + } + } + ); + } + return whereText; + }; + + const where = translateInner(entity, './', filter); + if (extraWhere && where) { + return `${extraWhere} and ${where}`; + } + return extraWhere || where; + } + + private translateSorter(entity: T, sorter: ED[T]['Selection']['sorter'], aliasDict: Record): string { + const translateInner = (entity2: E, sortAttr: DeduceSorterAttr, path: string): string => { + assert(Object.keys(sortAttr).length === 1); + const attr = Object.keys(sortAttr)[0]; + const alias = aliasDict[path]; + + if (attr.toLocaleLowerCase().startsWith(EXPRESSION_PREFIX)) { + return this.translateExpression(alias, sortAttr[attr] as any, {}); + } + else if (sortAttr[attr] === 1) { + return `\`${alias}\`.\`${attr}\``; + } + else { + const rel = judgeRelation(this.schema, entity2, attr); + if (typeof rel === 'string') { + return translateInner(rel, sortAttr[attr] as any, `${path}${attr}/`); + } + else { + assert(rel === 2); + return translateInner(attr, sortAttr[attr] as any, `${path}${attr}/`); + } + } + }; + + let sortText = ''; + sorter!.forEach( + (sortNode, index) => { + const { $attr, $direction } = sortNode; + sortText += translateInner(entity, $attr, './'); + if ($direction) { + sortText += ` ${$direction}`; + } + + if (index < sorter!.length - 1) { + sortText += ','; + } + } + ); + + return sortText; + } + + private translateProjection( + entity: T, + projection: ED[T]['Selection']['data'], + aliasDict: Record, + projectionRefAlias: Record): string { + const { schema } = this; + const translateInner = (entity2: E, projection2: ED[E]['Selection']['data'], path: string): string => { + const alias = aliasDict[path]; + const { attributes } = schema[entity2]; + let projText = ''; + + let prefix = path.slice(2).replace(/\//g, '.'); + Object.keys(projection2).forEach( + (attr, idx) => { + if (attr.toLowerCase().startsWith(EXPRESSION_PREFIX)) { + const exprText = this.translateExpression(alias, projection2[attr], projectionRefAlias); + projText += ` ${exprText} as ${prefix}${attr}`; + } + else { + const rel = judgeRelation(this.schema, entity2, attr); + if (typeof rel === 'string') { + projText += translateInner(rel, projection2[attr], `${path}${attr}/`); + } + else if (rel === 2) { + projText += translateInner(attr, projection2[attr], `${path}${attr}/`); + } + else { + assert(rel === 0 || rel === 1); + const { type } = attributes[attr]; + if (projection2[attr] === 1) { + projText += ` ${this.translateAttrProjection(type as DataType, alias, attr)} as \`${prefix}${attr}\``; + } + else { + assert (typeof projection2 === 'string'); + projText += ` ${this.translateAttrProjection(type as DataType, alias, attr)} as \`${prefix}${projection2[attr]}\``; + } + } + } + if (idx < Object.keys(projection2).length - 1) { + projText += ','; + } + } + ); + + return projText; + }; + + return translateInner(entity, projection, './'); + } + + translateSelect(entity: T, selection: ED[T]['Selection'], params?: SelectParams): string { + const { data, filter, sorter, indexFrom, count } = selection; + const { from: fromText, aliasDict, projectionRefAlias, extraWhere, filterRefAlias } = this.analyzeJoin(entity, { + projection: data, + filter, + sorter, + }); + + const projText = this.translateProjection(entity, data, aliasDict, projectionRefAlias); + + const filterText = this.translateFilter(entity, aliasDict, filterRefAlias, filter, extraWhere); + + const sorterText = sorter && this.translateSorter(entity, sorter, aliasDict) ; + + return this.populateSelectStmt(projText, fromText, aliasDict, filterText, sorterText, indexFrom, count, params); + } + + translateRemove(entity: T, operation: ED[T]['Remove'], params?: SelectParams): string { + const { filter, sorter, indexFrom, count } = operation; + const { aliasDict, filterRefAlias, extraWhere, from: fromText } = this.analyzeJoin(entity, { filter, sorter }); + + const alias = aliasDict['./']; + + const filterText = this.translateFilter(entity, aliasDict, filterRefAlias, filter, extraWhere); + + const sorterText = sorter && sorter.length > 0 ? this.translateSorter(entity, sorter, aliasDict) : undefined; + + return this.populateRemoveStmt(alias, fromText, aliasDict, filterText, sorterText, indexFrom, count, params); + } + + translateUpdate(entity: T, operation: ED[T]['Update'], params?: any): string { + const { attributes } = this.schema[entity]; + const { filter, sorter, indexFrom, count, data } = operation; + const { aliasDict, filterRefAlias, extraWhere, from: fromText } = this.analyzeJoin(entity, { filter, sorter }); + + const alias = aliasDict['./']; + + let updateText = ''; + for (const attr in data) { + if (updateText) { + updateText += ','; + } + assert(attributes.hasOwnProperty(attr) && attributes[attr].type !== 'ref'); + const value = this.translateAttrValue(attributes[attr].type as DataType, data[attr]); + updateText += `\`${alias}\`.\`${attr}\` = ${value}`; + } + + const filterText = this.translateFilter(entity, aliasDict, filterRefAlias, filter, extraWhere); + const sorterText = sorter && this.translateSorter(entity, sorter, aliasDict); + + return this.populateUpdateStmt(updateText, fromText, aliasDict, filterText, sorterText, indexFrom, count, params); + } + + translateDestroyEntity(entity: string, truncate?: boolean): string { + const { schema } = this; + const { storageName = entity, view } = schema[entity]; + + let sql: string; + if (view) { + sql = `drop view if exists \`${storageName}\``; + } + else { + sql = truncate ? `truncate table \`${storageName}\`` : `drop table if exists \`${storageName}\``; + } + + return sql; + } + + + escapeStringValue(value: string): string { + const result = `'${value.replace(/'/g, '\\\'')}'`; + return result; + } +} \ No newline at end of file diff --git a/test/entities/House.ts b/test/entities/House.ts new file mode 100644 index 0000000..0957ec7 --- /dev/null +++ b/test/entities/House.ts @@ -0,0 +1,24 @@ +import { String, Int, Datetime, Image, Boolean, Text } from 'oak-domain/lib/types/DataType'; +import { Schema as Area } from 'oak-general-business/lib/entities/Area'; +import { Schema as User } from 'oak-general-business/lib/entities/User'; +import { Schema as ExtraFile } from 'oak-general-business/lib/entities/ExtraFile'; +import { EntityShape } from 'oak-domain/lib/types'; +import { LocaleDef } from 'oak-domain/lib/types/Locale'; + +export interface Schema extends EntityShape { + district: String<16>; + area: Area; + owner: User; + dd: Array; +}; + +const locale: LocaleDef = { + zh_CN: { + attr: { + district: '街区', + area: '地区', + owner: '房主', + dd: '文件' + }, + }, +}; \ No newline at end of file diff --git a/test/testSqlTranslator.ts b/test/testSqlTranslator.ts new file mode 100644 index 0000000..2199ff5 --- /dev/null +++ b/test/testSqlTranslator.ts @@ -0,0 +1,43 @@ +import { MySqlTranslator } from '../src/MySQL/translator'; +import { EntityDict, storageSchema } from './test-app-domain'; + +describe('test MysqlTranslator', function() { + this.timeout(100000); + let translator: MySqlTranslator; + before(() => { + translator = new MySqlTranslator(storageSchema); + }); + + it('test create', () => { + const sql = translator.translateCreateEntity('token'); + console.log(sql); + }); + + it('test insert', () => { + const sql = translator.translateInsert('user', [{ + id: 'xcxc', + name: 'xc', + nickname: 'xcxcxc', + }, { + id: 'gggg', + name: 'gg', + nickname: 'gggggg', + }]); + // console.log(sql); + }); + + it('test select', () => { + const sql = translator.translateSelect('token', { + data: { + id: 1, + $$createAt$$: 1, + userId: 1, + mobile: { + id: 1, + mobile: 1, + }, + }, + }); + // console.log(sql); + }) +}); \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..27b7cd5 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "target": "esnext", + "allowJs": false, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "strict": true, + "skipLibCheck": true, + "lib": [ + "ES2020", + "DOM" + ], + "outDir": "lib", /* Redirect output structure to the directory. */ + "rootDir": "src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + "types": [ + "node", + "mocha" + ], + "forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */ + "resolveJsonModule": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} \ No newline at end of file