部分ddl的实现

src/MySQL/store.ts

@@ -1,10 +1,10 @@
-import { EntityDict, OperateOption, OperationResult, TxnOption, StorageSchema, SelectOption, AggregationResult, Geo } from 'oak-domain/lib/types';
+import { EntityDict, OperateOption, OperationResult, TxnOption, StorageSchema, SelectOption, AggregationResult, Geo, Attributes, Attribute, Index, IndexConfig } from 'oak-domain/lib/types';
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { CascadeStore } from 'oak-domain/lib/store/CascadeStore';
 import { MySQLConfiguration } from './types/Configuration';
 import { MySqlConnector } from './connector';
 import { MySqlTranslator, MySqlSelectOption, MysqlOperateOption } from './translator';
-import { assign, set } from 'lodash';
+import { assign, difference, set, pick } from 'lodash';
 import assert from 'assert';
 import { judgeRelation } from 'oak-domain/lib/store/relation';
 import { AsyncContext, AsyncRowStore } from 'oak-domain/lib/store/AsyncRowStore';
@@ -30,7 +30,7 @@ function convertGeoTextToObject(geoText: string): Geo {
     }
 }
 
-export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends AsyncContext<ED>> extends CascadeStore<ED> implements AsyncRowStore<ED, Cxt>{
+export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends AsyncContext<ED>> extends CascadeStore<ED> implements AsyncRowStore<ED, Cxt> {
     protected countAbjointRow<T extends keyof ED, OP extends SelectOption, Cxt extends SyncContext<ED>>(entity: T, selection: Pick<ED[T]['Selection'], 'filter' | 'count'>, context: Cxt, option: OP): number {
         throw new Error('MySQL store不支持同步取数据，不应该跑到这儿');
     }
@@ -72,25 +72,25 @@ export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends Asyn
     }
     private formResult<T extends keyof ED>(entity: T, result: any): any {
         const schema = this.getSchema();
-       /*  function resolveObject(r: Record<string, any>, path: string, value: any) {
+        /*  function resolveObject(r: Record<string, any>, path: string, value: any) {
-            const i = path.indexOf(".");
+             const i = path.indexOf(".");
-            const bs = path.indexOf('[');
+             const bs = path.indexOf('[');
-            const be = path.indexOf(']');
+             const be = path.indexOf(']');
-            if (i === -1 && bs === -1) {
+             if (i === -1 && bs === -1) {
-                r[i] = value;
+                 r[i] = value;
-            }
+             }
-            else if (i === -1) {
+             else if (i === -1) {
-
+ 
-            }
+             }
-            else if (bs === -1) {
+             else if (bs === -1) {
-                const attrHead = path.slice(0, i);
+                 const attrHead = path.slice(0, i);
-                const attrTail = path.slice(i + 1);
+                 const attrTail = path.slice(i + 1);
-                if (!r[attrHead]) {
+                 if (!r[attrHead]) {
-                    r[attrHead] = {};
+                     r[attrHead] = {};
-                }
+                 }
-                resolveObject(r[attrHead], attrTail, value);
+                 resolveObject(r[attrHead], attrTail, value);
-            }
+             }
-        } */
+         } */
         function resolveAttribute<E extends keyof ED>(entity2: E, r: Record<string, any>, attr: string, value: any) {
             const { attributes, view } = schema[entity2];
             if (!view) {
@@ -335,4 +335,146 @@ export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends Asyn
             }
         }
     }
-}
+
+    // 从数据库中读取当前schema
+    readSchema() {
+        return this.translator.readSchema((sql) => this.connector.exec(sql));
+    }
+
+    /**
+     * 根据载入的dataSchema，和数据库中原来的schema，决定如何来upgrade
+     * 制订出来的plan分为两阶段：增加阶段和削减阶段，在两个阶段之间，由用户来修正数据
+     */
+    async makeUpgradePlan() {
+        const originSchema = await this.readSchema();
+        const plan = this.diffSchema(originSchema, this.translator.schema);
+        return plan;
+    }
+
+    /**
+     * 比较两个schema的不同，这里计算的是new对old的增量
+     * @param schemaOld 
+     * @param SchemaNew 
+     */
+    diffSchema(schemaOld: StorageSchema<any>, schemaNew: StorageSchema<any>) {
+        const plan: Plan = {
+            newTables: {},
+            newIndexes: {},
+            updatedIndexes: {},
+            updatedTables: {},
+        };
+
+        for (const table in schemaNew) {
+            // mysql数据字典不分大小写的
+            if (schemaOld[table] || schemaOld[table.toLowerCase()]) {
+                const { attributes, indexes } = schemaOld[table] || schemaOld[table.toLowerCase()];
+                const { attributes: attributesNew, indexes: indexesNew } = schemaNew[table];
+
+                const assignToUpdateTables = (attr: string, isNew: boolean) => {
+                    if (!plan.updatedTables[table]) {
+                        plan.updatedTables[table] = {
+                            attributes: {
+                                [attr]: {
+                                    ...attributesNew[attr],
+                                    isNew,
+                                }
+                            }
+                        };
+                    }
+                    else {
+                        plan.updatedTables[table].attributes[attr] = {
+                            ...attributesNew[attr],
+                            isNew,
+                        };
+                    }
+                };
+                for (const attr in attributesNew) {
+                    if (attributes[attr]) {
+                        // 因为反向无法复原原来定义的attribute类型，这里就比较两次创建的sql是不是一致。
+                        const sql1 = this.translator.translateAttributeDef(attr, attributesNew[attr]);
+                        const sql2 = this.translator.translateAttributeDef(attr, attributes[attr]);
+                        if (!this.translator.compareSql(sql1, sql2)) {
+                            assignToUpdateTables(attr, false);
+                        }
+                    }
+                    else {
+                        assignToUpdateTables(attr, true);
+                    }
+                }
+
+                if (indexesNew) {
+                    const assignToIndexes = (index: Index<any>, isNew: boolean) => {
+                        if (isNew) {
+                            if (plan.newIndexes[table]) {
+                                plan.newIndexes[table].push(index);
+                            }
+                            else {
+                                plan.newIndexes[table] = [index];
+                            }
+                        }
+                        else {
+                            if (plan.updatedIndexes[table]) {
+                                plan.updatedIndexes[table].push(index);
+                            }
+                            else {
+                                plan.updatedIndexes[table] = [index];
+                            }
+                        }
+                    };
+                    const compareConfig = (config1?: IndexConfig, config2?: IndexConfig) => {
+                        const unique1 = config1?.unique || false;
+                        const unique2 = config2?.unique || false;
+                        if (unique1 !== unique2) {
+                            return false;
+                        }
+                        const type1 = config1?.type || 'btree';
+                        const type2 = config2?.type || 'btree';
+
+                        // parser目前无法从mysql中读出来，所以不比了
+                        return type1 === type2;
+                    };
+                    for (const index of indexesNew) {
+                        const { name, config, attributes } = index;
+                        const origin = indexes?.find(ele => ele.name === name);
+                        if (origin) {
+                            if (JSON.stringify(attributes) !== JSON.stringify(origin.attributes)) {
+                                // todo，这里要细致比较，不能用json.stringify
+                                assignToIndexes(index, false);
+                            }
+                            else {
+                                if (!compareConfig(config, origin.config)) {
+                                    assignToIndexes(index, false);
+                                }
+                            }
+                        }
+                        else {
+                            assignToIndexes(index, true);
+                        }
+                    }
+                }
+            }
+            else {
+                plan.newTables[table] = {
+                    attributes: schemaNew[table].attributes,
+                };
+                if (schemaNew[table].indexes) {
+                    plan.newIndexes[table] = schemaNew[table].indexes!;
+                }
+            }
+        }
+
+        return plan;
+    }
+}
+
+
+type Plan = {
+    newTables: Record<string, {
+        attributes: Record<string, Attribute>;
+    }>;
+    newIndexes: Record<string, Index<any>[]>;
+    updatedTables: Record<string, {
+        attributes: Record<string, Attribute & { isNew: boolean }>;
+    }>;
+    updatedIndexes: Record<string, Index<any>[]>;
+};

src/MySQL/translator.ts

@@ -1,7 +1,7 @@
 import assert from 'assert';
 import { format } from 'util';
-import { assign } from 'lodash';
+import { assign, groupBy } from 'lodash';
-import { EntityDict, Geo, Q_FullTextValue, RefOrExpression, Ref, StorageSchema, Index, RefAttr, DeleteAtAttribute } from "oak-domain/lib/types";
+import { EntityDict, Geo, Q_FullTextValue, RefOrExpression, Ref, StorageSchema, Index, RefAttr, DeleteAtAttribute, Attributes, Attribute, PrimaryKeyAttribute } from "oak-domain/lib/types";
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { DataType, DataTypeParams } from "oak-domain/lib/types/schema/DataTypes";
 import { SqlOperateOption, SqlSelectOption, SqlTranslator } from "../sqlTranslator";
@@ -308,14 +308,18 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
             return 'text ';
         }
         if (type === 'ref') {
-            return 'char(36)';
+            return 'char(36) ';
+        }
+        if (['bool', 'boolean'].includes(type)) {
+            // MySQL读出来就是tinyint(1)
+            return 'tinyint(1) ';
         }
         if (type === 'money') {
-            return 'bigint';
+            return 'bigint ';
         }
         if (type === 'enum') {
             assert(enumeration);
-            return `enum(${enumeration.map(ele => `'${ele}'`).join(',')})`;
+            return `enum(${enumeration.map(ele => `'${ele}'`).join(',')}) `;
         }
 
         if (MySqlTranslator.withLengthDataTypes.includes(type as DataType)) {
@@ -335,35 +339,35 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
                 if (typeof scale === 'number') {
                     return `${type}(${precision}, ${scale}) `;
                 }
-                return `${type}(${precision})`;
+                return `${type}(${precision}) `;
             }
             else {
                 const { precision, scale } = (MySqlTranslator.dataTypeDefaults as any)[type];
                 if (typeof scale === 'number') {
                     return `${type}(${precision}, ${scale}) `;
                 }
-                return `${type}(${precision})`;
+                return `${type}(${precision}) `;
             }
         }
 
         if (MySqlTranslator.withWidthDataTypes.includes(type as DataType)) {
             assert(type === 'int');
-            const { width } = params!;
+            const { width } = params || { width: 4 };
             switch (width!) {
                 case 1: {
-                    return 'tinyint';
+                    return 'tinyint ';
                 }
                 case 2: {
-                    return 'smallint';
+                    return 'smallint ';
                 }
                 case 3: {
-                    return 'mediumint';
+                    return 'mediumint ';
                 }
                 case 4: {
-                    return 'int';
+                    return 'int ';
                 }
                 default: {
-                    return 'bigint';
+                    return 'bigint ';
                 }
             }
         }
@@ -468,7 +472,7 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
                             }
                         }
                         else {
-                            assert (typeof length === 'object');
+                            assert(typeof length === 'object');
                             const op = Object.keys(length)[0];
                             assert(op.startsWith('$'));
                             if (p) {
@@ -617,6 +621,40 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
         );
         return ` match(${columns2.join(',')}) against ('${$search}' in natural language mode)`;
     }
+
+    translateAttributeDef(attr: string, attrDef: Attribute) {
+        let sql = `\`${attr}\` `;
+        const {
+            type,
+            params,
+            default: defaultValue,
+            unique,
+            notNull,
+            sequenceStart,
+            enumeration,
+        } = attrDef;
+        sql += this.populateDataTypeDef(type, params, enumeration) as string;
+
+        if (notNull || type === 'geometry') {
+            sql += ' not null ';
+        }
+        if (unique) {
+            sql += ' unique ';
+        }
+        if (typeof sequenceStart === 'number') {
+            sql += ' auto_increment unique ';
+        }
+        if (defaultValue !== undefined) {
+            assert(type !== 'ref');
+            sql += ` default ${this.translateAttrValue(type, defaultValue)}`;
+        }
+        if (attr === PrimaryKeyAttribute) {
+            sql += ' primary key'
+        }
+
+        return sql;
+    }
+
     translateCreateEntity<T extends keyof ED>(entity: T, options?: CreateEntityOption): string[] {
         const ifExists = options?.ifExists || 'drop';
         const { schema } = this;
@@ -647,41 +685,14 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
             // 翻译所有的属性
             Object.keys(attributes).forEach(
                 (attr, idx) => {
-                    const attrDef = attributes[attr];
+                    const attrSql = this.translateAttributeDef(attr, attributes[attr]);
-                    const {
+                    if (idx !== 0) {
-                        type,
+                        sql +=', ';
-                        params,
-                        default: defaultValue,
-                        unique,
-                        notNull,
-                        sequenceStart,
-                        enumeration,
-                    } = attrDef;
-                    sql += `\`${attr}\` `
-                    sql += this.populateDataTypeDef(type, params, enumeration) as string;
-
-                    if (notNull || type === 'geometry') {
-                        sql += ' not null ';
                     }
-                    if (unique) {
+                    sql += attrSql;
-                        sql += ' unique ';
+                    if (typeof attributes[attr].sequenceStart === 'number') {
-                    }
+                        assert(hasSequence === false, 'Entity can only have one auto increment attribute.');
-                    if (sequenceStart) {
+                        hasSequence = attributes[attr].sequenceStart!;
-                        if (hasSequence) {
-                            throw new Error(`「${entity as string}」只能有一个sequence列`);
-                        }
-                        hasSequence = sequenceStart;
-                        sql += ' auto_increment unique ';
-                    }
-                    if (defaultValue !== undefined) {
-                        assert(type !== 'ref');
-                        sql += ` default ${this.translateAttrValue(type, defaultValue)}`;
-                    }
-                    if (attr === 'id') {
-                        sql += ' primary key'
-                    }
-                    if (idx < Object.keys(attributes).length - 1) {
-                        sql += ',\n';
                     }
                 }
             );
@@ -702,13 +713,12 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
                         else if (type === 'spatial') {
                             sql += ' spatial ';
                         }
-                        sql += `index ${name} `;
+                        sql += `index \`${name}\` `;
                         if (type === 'hash') {
                             sql += ` using hash `;
                         }
                         sql += '(';
 
-                        let includeDeleteAt = false;
                         attributes.forEach(
                             ({ name, size, direction }, idx2) => {
                                 sql += `\`${name as string}\``;
@@ -719,16 +729,10 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
                                     sql += ` ${direction}`;
                                 }
                                 if (idx2 < attributes.length - 1) {
-                                    sql += ','
+                                    sql += ', ';
-                                }
-                                if (name === '$$deleteAt$$') {
-                                    includeDeleteAt = true;
                                 }
                             }
                         );
-                        if (!includeDeleteAt && !type) {
-                            sql += ', `$$deleteAt$$`'; // 在mysql80+之后，需要给属性加上``包裹，否则会报错
-                        }
                         sql += ')';
                         if (parser) {
                             sql += ` with parser ${parser}`;
@@ -1077,4 +1081,135 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
 
         return sql;
     }
+
+    /**
+     * 将MySQL返回的Type回译成oak的类型，是 populateDataTypeDef 的反函数
+     * @param type 
+     */
+    private reTranslateToAttribute(type: string): Attribute {
+        const withLengthDataTypes = MySqlTranslator.withLengthDataTypes.join('|')
+        let result = (new RegExp(`^(${withLengthDataTypes})\\((\\d+)\\)$`)).exec(type);
+        if (result) {
+            return {
+                type: result[1] as DataType,
+                params: {
+                    length: parseInt(result[2]),
+                }
+            };
+        }
+
+        const withPrecisionDataTypes = MySqlTranslator.withPrecisionDataTypes.join('|')
+        result = (new RegExp(`^(${withPrecisionDataTypes})\\((\\d+),(d+)\\)$`)).exec(type);
+        if (result) {
+            return {
+                type: result[1] as DataType,
+                params: {
+                    precision: parseInt(result[2]),
+                    scale: parseInt(result[3]),
+                },
+            };
+        }
+
+        result = (/^enum\((\S+)\)$/).exec(type);
+        if (result) {
+            const enumeration = result[1].split(',').map(
+                ele => ele.slice(1, -1)
+            );
+            return {
+                type: 'enum',
+                enumeration
+            };
+        }
+
+        return {
+            type: type as DataType,
+        };
+    }
+
+    // 分析当前数据库结构图
+    async readSchema(execFn: (sql: string) => Promise<any>) {
+        const result: (typeof this.schema) = {} as typeof this.schema;
+        const sql = 'show tables;';
+        const [tables] = await execFn(sql);
+        for (const tableItem of tables) {
+            const table = Object.values(tableItem)[0] as string;
+            const [tableResult] = await execFn(`desc \`${table}\``);
+
+            const attributes: Attributes<any> = {};
+            for (const attrItem of tableResult) {
+                const { Field: attrName, Null: isNull, Type: type, Key: key } = attrItem as {
+                    Field: string,
+                    Null: 'YES' | 'NO',
+                    Type: string,
+                    Key: 'UNI' | 'MUL',
+                    Extra: string;
+                };
+                attributes[attrName] = {
+                    ...this.reTranslateToAttribute(type),
+                    notNull: isNull.toUpperCase() === 'NO',
+                    unique: key.toUpperCase() === 'UNI',
+                };
+                // 自增列只可能是seq
+                if (attrName === '$$seq$$') {
+                    attributes[attrName].sequenceStart = 10000;
+                }
+            }
+            Object.assign(result, {
+                [table]: {
+                    attributes,
+                }
+            });
+
+            const [indexedColumns] = (await execFn(`show index from \`${table}\``)) as [
+                Array<{
+                    Non_unique: 0 | 1;
+                    Key_name: string;
+                    Seq_in_index: number;
+                    Column_name: string;
+                    Index_type: string;
+                    Null: string;
+                    Collation: 'A' | 'D';
+                    Sub_part: number;
+                }>
+            ];
+            if (indexedColumns.length) {
+                const groupedColumns = groupBy(
+                    indexedColumns.sort(
+                        (ele1, ele2) => ele1.Key_name.localeCompare(ele2.Key_name) || ele1.Seq_in_index - ele2.Seq_in_index
+                    ),
+                    'Key_name'
+                );
+
+                const indexes = Object.values(groupedColumns).map(
+                    (ele) => {
+                        const index: Index<any> = {
+                            name: ele[0].Key_name,
+                            attributes: ele.map(ele2 => ({
+                                name: ele2.Column_name,
+                                direction: ele2.Collation === 'D' ? 'DESC' : (ele2.Collation === 'A' ? 'ASC' : undefined),
+                                size: ele2.Sub_part || undefined,
+                            })),
+                        };
+                        if (ele[0].Non_unique === 0 || ele[0].Index_type.toUpperCase() !== 'BTREE') {
+                            index.config = {};
+                            if (ele[0].Non_unique === 0) {
+                                index.config.unique = true;
+                            }
+                            if (ele[0].Index_type.toUpperCase() !== 'BTREE') {
+                                index.config.type = ele[0].Index_type.toLowerCase() as any;
+                            }
+                        }
+
+                        return index;
+                    }
+                );
+
+                Object.assign(result[table], {
+                    indexes,
+                });
+            }
+        }
+
+        return result;
+    }
 }

src/sqlTranslator.ts

@@ -4,7 +4,8 @@ import { assign, cloneDeep, difference, identity, intersection, keys, set } from
 import {
     Attribute, EntityDict, EXPRESSION_PREFIX, Index, OperateOption,
     Q_FullTextValue, Ref, RefOrExpression, SelectOption, StorageSchema, SubQueryPredicateMetadata,
-    TriggerDataAttribute, CreateAtAttribute, UpdateAtAttribute, DeleteAtAttribute, SeqAttribute, TriggerUuidAttribute
+    TriggerDataAttribute, CreateAtAttribute, UpdateAtAttribute, DeleteAtAttribute, SeqAttribute, TriggerUuidAttribute,
+    PrimaryKeyAttribute
 } from "oak-domain/lib/types";
 import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { DataType } from "oak-domain/lib/types/schema/DataTypes";
@@ -29,7 +30,7 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
             const { attributes, indexes } = schema[entity];
             // 增加默认的属性
             assign(attributes, {
-                id: {
+                [PrimaryKeyAttribute]: {
                     type: 'char',
                     params: {
                         length: 36,
@@ -78,7 +79,7 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
                         name: DeleteAtAttribute,
                     }],
                 }, {
-                    name: `${entity}_trigger_uuid`,
+                    name: `${entity}_trigger_uuid_auto_create`,
                     attributes: [{
                         name: TriggerUuidAttribute,
                     }]
@@ -159,6 +160,18 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
             }
 
             if (indexes) {
+                for (const index of indexes) {
+                    const { attributes, config } = index;
+                    if (!config?.type || config.type === 'btree') {
+                        if (!attributes.find(
+                            (ele) => ele.name === DeleteAtAttribute
+                        )) {
+                            attributes.push({
+                                name: DeleteAtAttribute,
+                            });
+                        }
+                    }
+                }
                 indexes.push(...intrinsticIndexes);
             }
             else {
@@ -1235,4 +1248,12 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
         const result = SqlString.escape(value);
         return result;
     }
+
+
+    /**比较两段sql是否完全一致，这里是把所有的空格去掉了 */
+    compareSql(sql1: string, sql2: string) {
+        const reg = /[\t\r\f\n\s]/g;
+
+        return sql1.replaceAll(reg, '') === sql2.replaceAll(reg, '');
+    }
 }

test/testMySQLStore.ts

@@ -1735,6 +1735,24 @@ describe('test mysqlstore', function () {
                 }
             }
         }, context, {});
+        assert (typeof (row[0]?.data as any).price === 'string');
+
+        // todo 这个暂时还没搞定。需求也不是太强
+        const row1 = await store.select('oper', {
+            data: {
+                id: 1,
+                data: {
+                    name: 1,
+                    price: 2,
+                },
+            },
+            filter: {
+                data: {
+                    price: [undefined, 400],
+                }
+            }
+        }, context, {});
+        // assert (typeof (row[0]?.data as any).price === 'object');
 
         const row2 = await store.select('oper', {
             data: {
@@ -2223,6 +2241,19 @@ describe('test mysqlstore', function () {
         assert(r8.map(ele => ele.id).includes(id3));
     });
 
+    it('[2.0.1]read schema', async () => {
+        const result = await store.readSchema();
+        
+        console.log(result);
+    });
+
+    it('[2.0.2]diff schema', async () => {
+        const result = await store.makeUpgradePlan();
+
+        console.log(result);
+    });
+
+
     after(() => {
         store.disconnect();
     });