Oak-Team
/
oak-db
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 13 Wiki Activity

部分ddl的实现

This commit is contained in:
Xu Chang 2026-01-01 13:25:38 +08:00
parent aade97762f
commit 1a3e3cb005
4 changed files with 412 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
src/MySQL/store.ts
View File

@ -1,10 +1,10 @@
import { EntityDict, OperateOption, OperationResult, TxnOption, StorageSchema, SelectOption, AggregationResult, Geo } from 'oak-domain/lib/types';
import { EntityDict, OperateOption, OperationResult, TxnOption, StorageSchema, SelectOption, AggregationResult, Geo, Attributes, Attribute, Index, IndexConfig } from 'oak-domain/lib/types';
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
import { CascadeStore } from 'oak-domain/lib/store/CascadeStore';
import { MySQLConfiguration } from './types/Configuration';
import { MySqlConnector } from './connector';
import { MySqlTranslator, MySqlSelectOption, MysqlOperateOption } from './translator';
import { assign, set } from 'lodash';
import { assign, difference, set, pick } from 'lodash';
import assert from 'assert';
import { judgeRelation } from 'oak-domain/lib/store/relation';
import { AsyncContext, AsyncRowStore } from 'oak-domain/lib/store/AsyncRowStore';
@ -30,7 +30,7 @@ function convertGeoTextToObject(geoText: string): Geo {
}
}
export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends AsyncContext<ED>> extends CascadeStore<ED> implements AsyncRowStore<ED, Cxt>{
export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends AsyncContext<ED>> extends CascadeStore<ED> implements AsyncRowStore<ED, Cxt> {
protected countAbjointRow<T extends keyof ED, OP extends SelectOption, Cxt extends SyncContext<ED>>(entity: T, selection: Pick<ED[T]['Selection'], 'filter' | 'count'>, context: Cxt, option: OP): number {
throw new Error('MySQL store不支持同步取数据，不应该跑到这儿');
}
@ -72,25 +72,25 @@ export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends Asyn
}
private formResult<T extends keyof ED>(entity: T, result: any): any {
const schema = this.getSchema();
/* function resolveObject(r: Record<string, any>, path: string, value: any) {
const i = path.indexOf(".");
const bs = path.indexOf('[');
const be = path.indexOf(']');
if (i === -1 && bs === -1) {
r[i] = value;
}
else if (i === -1) {
}
else if (bs === -1) {
const attrHead = path.slice(0, i);
const attrTail = path.slice(i + 1);
if (!r[attrHead]) {
r[attrHead] = {};
}
resolveObject(r[attrHead], attrTail, value);
}
} */
/* function resolveObject(r: Record<string, any>, path: string, value: any) {
const i = path.indexOf(".");
const bs = path.indexOf('[');
const be = path.indexOf(']');
if (i === -1 && bs === -1) {
r[i] = value;
}
else if (i === -1) {
}
else if (bs === -1) {
const attrHead = path.slice(0, i);
const attrTail = path.slice(i + 1);
if (!r[attrHead]) {
r[attrHead] = {};
}
resolveObject(r[attrHead], attrTail, value);
}
} */
function resolveAttribute<E extends keyof ED>(entity2: E, r: Record<string, any>, attr: string, value: any) {
const { attributes, view } = schema[entity2];
if (!view) {
@ -335,4 +335,146 @@ export class MysqlStore<ED extends EntityDict & BaseEntityDict, Cxt extends Asyn
}
}
}
}
// 从数据库中读取当前schema
readSchema() {
return this.translator.readSchema((sql) => this.connector.exec(sql));
}
/**
* 根据载入的dataSchema，和数据库中原来的schema，决定如何来upgrade
* 制订出来的plan分为两阶段：增加阶段和削减阶段，在两个阶段之间，由用户来修正数据
*/
async makeUpgradePlan() {
const originSchema = await this.readSchema();
const plan = this.diffSchema(originSchema, this.translator.schema);
return plan;
}
/**
* 比较两个schema的不同，这里计算的是new对old的增量
* @param schemaOld
* @param SchemaNew
*/
diffSchema(schemaOld: StorageSchema<any>, schemaNew: StorageSchema<any>) {
const plan: Plan = {
newTables: {},
newIndexes: {},
updatedIndexes: {},
updatedTables: {},
};
for (const table in schemaNew) {
// mysql数据字典不分大小写的
if (schemaOld[table] || schemaOld[table.toLowerCase()]) {
const { attributes, indexes } = schemaOld[table] || schemaOld[table.toLowerCase()];
const { attributes: attributesNew, indexes: indexesNew } = schemaNew[table];
const assignToUpdateTables = (attr: string, isNew: boolean) => {
if (!plan.updatedTables[table]) {
plan.updatedTables[table] = {
attributes: {
[attr]: {
...attributesNew[attr],
isNew,
}
}
};
}
else {
plan.updatedTables[table].attributes[attr] = {
...attributesNew[attr],
isNew,
};
}
};
for (const attr in attributesNew) {
if (attributes[attr]) {
// 因为反向无法复原原来定义的attribute类型，这里就比较两次创建的sql是不是一致。
const sql1 = this.translator.translateAttributeDef(attr, attributesNew[attr]);
const sql2 = this.translator.translateAttributeDef(attr, attributes[attr]);
if (!this.translator.compareSql(sql1, sql2)) {
assignToUpdateTables(attr, false);
}
}
else {
assignToUpdateTables(attr, true);
}
}
if (indexesNew) {
const assignToIndexes = (index: Index<any>, isNew: boolean) => {
if (isNew) {
if (plan.newIndexes[table]) {
plan.newIndexes[table].push(index);
}
else {
plan.newIndexes[table] = [index];
}
}
else {
if (plan.updatedIndexes[table]) {
plan.updatedIndexes[table].push(index);
}
else {
plan.updatedIndexes[table] = [index];
}
}
};
const compareConfig = (config1?: IndexConfig, config2?: IndexConfig) => {
const unique1 = config1?.unique || false;
const unique2 = config2?.unique || false;
if (unique1 !== unique2) {
return false;
}
const type1 = config1?.type || 'btree';
const type2 = config2?.type || 'btree';
// parser目前无法从mysql中读出来，所以不比了
return type1 === type2;
};
for (const index of indexesNew) {
const { name, config, attributes } = index;
const origin = indexes?.find(ele => ele.name === name);
if (origin) {
if (JSON.stringify(attributes) !== JSON.stringify(origin.attributes)) {
// todo，这里要细致比较，不能用json.stringify
assignToIndexes(index, false);
}
else {
if (!compareConfig(config, origin.config)) {
assignToIndexes(index, false);
}
}
}
else {
assignToIndexes(index, true);
}
}
}
}
else {
plan.newTables[table] = {
attributes: schemaNew[table].attributes,
};
if (schemaNew[table].indexes) {
plan.newIndexes[table] = schemaNew[table].indexes!;
}
}
}
return plan;
}
}
type Plan = {
newTables: Record<string, {
attributes: Record<string, Attribute>;
}>;
newIndexes: Record<string, Index<any>[]>;
updatedTables: Record<string, {
attributes: Record<string, Attribute & { isNew: boolean }>;
}>;
updatedIndexes: Record<string, Index<any>[]>;
};

249
src/MySQL/translator.ts
View File

@ -1,7 +1,7 @@
import assert from 'assert';
import { format } from 'util';
import { assign } from 'lodash';
import { EntityDict, Geo, Q_FullTextValue, RefOrExpression, Ref, StorageSchema, Index, RefAttr, DeleteAtAttribute } from "oak-domain/lib/types";
import { assign, groupBy } from 'lodash';
import { EntityDict, Geo, Q_FullTextValue, RefOrExpression, Ref, StorageSchema, Index, RefAttr, DeleteAtAttribute, Attributes, Attribute, PrimaryKeyAttribute } from "oak-domain/lib/types";
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
import { DataType, DataTypeParams } from "oak-domain/lib/types/schema/DataTypes";
import { SqlOperateOption, SqlSelectOption, SqlTranslator } from "../sqlTranslator";
@ -308,14 +308,18 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
return 'text ';
}
if (type === 'ref') {
return 'char(36)';
return 'char(36) ';
}
if (['bool', 'boolean'].includes(type)) {
// MySQL读出来就是tinyint(1)
return 'tinyint(1) ';
}
if (type === 'money') {
return 'bigint';
return 'bigint ';
}
if (type === 'enum') {
assert(enumeration);
return `enum(${enumeration.map(ele => `'${ele}'`).join(',')})`;
return `enum(${enumeration.map(ele => `'${ele}'`).join(',')}) `;
}
if (MySqlTranslator.withLengthDataTypes.includes(type as DataType)) {
@ -335,35 +339,35 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
if (typeof scale === 'number') {
return `${type}(${precision}, ${scale}) `;
}
return `${type}(${precision})`;
return `${type}(${precision}) `;
}
else {
const { precision, scale } = (MySqlTranslator.dataTypeDefaults as any)[type];
if (typeof scale === 'number') {
return `${type}(${precision}, ${scale}) `;
}
return `${type}(${precision})`;
return `${type}(${precision}) `;
}
}
if (MySqlTranslator.withWidthDataTypes.includes(type as DataType)) {
assert(type === 'int');
const { width } = params!;
const { width } = params || { width: 4 };
switch (width!) {
case 1: {
return 'tinyint';
return 'tinyint ';
}
case 2: {
return 'smallint';
return 'smallint ';
}
case 3: {
return 'mediumint';
return 'mediumint ';
}
case 4: {
return 'int';
return 'int ';
}
default: {
return 'bigint';
return 'bigint ';
}
}
}
@ -468,7 +472,7 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
}
}
else {
assert (typeof length === 'object');
assert(typeof length === 'object');
const op = Object.keys(length)[0];
assert(op.startsWith('$'));
if (p) {
@ -617,6 +621,40 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
);
return ` match(${columns2.join(',')}) against ('${$search}' in natural language mode)`;
}
translateAttributeDef(attr: string, attrDef: Attribute) {
let sql = `\`${attr}\` `;
const {
type,
params,
default: defaultValue,
unique,
notNull,
sequenceStart,
enumeration,
} = attrDef;
sql += this.populateDataTypeDef(type, params, enumeration) as string;
if (notNull || type === 'geometry') {
sql += ' not null ';
}
if (unique) {
sql += ' unique ';
}
if (typeof sequenceStart === 'number') {
sql += ' auto_increment unique ';
}
if (defaultValue !== undefined) {
assert(type !== 'ref');
sql += ` default ${this.translateAttrValue(type, defaultValue)}`;
}
if (attr === PrimaryKeyAttribute) {
sql += ' primary key'
}
return sql;
}
translateCreateEntity<T extends keyof ED>(entity: T, options?: CreateEntityOption): string[] {
const ifExists = options?.ifExists || 'drop';
const { schema } = this;
@ -647,41 +685,14 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
// 翻译所有的属性
Object.keys(attributes).forEach(
(attr, idx) => {
const attrDef = attributes[attr];
const {
type,
params,
default: defaultValue,
unique,
notNull,
sequenceStart,
enumeration,
} = attrDef;
sql += `\`${attr}\` `
sql += this.populateDataTypeDef(type, params, enumeration) as string;
if (notNull || type === 'geometry') {
sql += ' not null ';
const attrSql = this.translateAttributeDef(attr, attributes[attr]);
if (idx !== 0) {
sql +=', ';
}
if (unique) {
sql += ' unique ';
}
if (sequenceStart) {
if (hasSequence) {
throw new Error(`「${entity as string}」只能有一个sequence列`);
}
hasSequence = sequenceStart;
sql += ' auto_increment unique ';
}
if (defaultValue !== undefined) {
assert(type !== 'ref');
sql += ` default ${this.translateAttrValue(type, defaultValue)}`;
}
if (attr === 'id') {
sql += ' primary key'
}
if (idx < Object.keys(attributes).length - 1) {
sql += ',\n';
sql += attrSql;
if (typeof attributes[attr].sequenceStart === 'number') {
assert(hasSequence === false, 'Entity can only have one auto increment attribute.');
hasSequence = attributes[attr].sequenceStart!;
}
}
);
@ -702,13 +713,12 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
else if (type === 'spatial') {
sql += ' spatial ';
}
sql += `index ${name} `;
sql += `index \`${name}\` `;
if (type === 'hash') {
sql += ` using hash `;
}
sql += '(';
let includeDeleteAt = false;
attributes.forEach(
({ name, size, direction }, idx2) => {
sql += `\`${name as string}\``;
@ -719,16 +729,10 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
sql += ` ${direction}`;
}
if (idx2 < attributes.length - 1) {
sql += ','
}
if (name === '$$deleteAt$$') {
includeDeleteAt = true;
sql += ', ';
}
}
);
if (!includeDeleteAt && !type) {
sql += ', `$$deleteAt$$`'; // 在mysql80+之后，需要给属性加上``包裹，否则会报错
}
sql += ')';
if (parser) {
sql += ` with parser ${parser}`;
@ -1077,4 +1081,135 @@ export class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends Sql
return sql;
}
/**
* 将MySQL返回的Type回译成oak的类型，是 populateDataTypeDef 的反函数
* @param type
*/
private reTranslateToAttribute(type: string): Attribute {
const withLengthDataTypes = MySqlTranslator.withLengthDataTypes.join('|')
let result = (new RegExp(`^(${withLengthDataTypes})\\((\\d+)\\)$`)).exec(type);
if (result) {
return {
type: result[1] as DataType,
params: {
length: parseInt(result[2]),
}
};
}
const withPrecisionDataTypes = MySqlTranslator.withPrecisionDataTypes.join('|')
result = (new RegExp(`^(${withPrecisionDataTypes})\\((\\d+),(d+)\\)$`)).exec(type);
if (result) {
return {
type: result[1] as DataType,
params: {
precision: parseInt(result[2]),
scale: parseInt(result[3]),
},
};
}
result = (/^enum\((\S+)\)$/).exec(type);
if (result) {
const enumeration = result[1].split(',').map(
ele => ele.slice(1, -1)
);
return {
type: 'enum',
enumeration
};
}
return {
type: type as DataType,
};
}
// 分析当前数据库结构图
async readSchema(execFn: (sql: string) => Promise<any>) {
const result: (typeof this.schema) = {} as typeof this.schema;
const sql = 'show tables;';
const [tables] = await execFn(sql);
for (const tableItem of tables) {
const table = Object.values(tableItem)[0] as string;
const [tableResult] = await execFn(`desc \`${table}\``);
const attributes: Attributes<any> = {};
for (const attrItem of tableResult) {
const { Field: attrName, Null: isNull, Type: type, Key: key } = attrItem as {
Field: string,
Null: 'YES' | 'NO',
Type: string,
Key: 'UNI' | 'MUL',
Extra: string;
};
attributes[attrName] = {
...this.reTranslateToAttribute(type),
notNull: isNull.toUpperCase() === 'NO',
unique: key.toUpperCase() === 'UNI',
};
// 自增列只可能是seq
if (attrName === '$$seq$$') {
attributes[attrName].sequenceStart = 10000;
}
}
Object.assign(result, {
[table]: {
attributes,
}
});
const [indexedColumns] = (await execFn(`show index from \`${table}\``)) as [
Array<{
Non_unique: 0 | 1;
Key_name: string;
Seq_in_index: number;
Column_name: string;
Index_type: string;
Null: string;
Collation: 'A' | 'D';
Sub_part: number;
}>
];
if (indexedColumns.length) {
const groupedColumns = groupBy(
indexedColumns.sort(
(ele1, ele2) => ele1.Key_name.localeCompare(ele2.Key_name) || ele1.Seq_in_index - ele2.Seq_in_index
),
'Key_name'
);
const indexes = Object.values(groupedColumns).map(
(ele) => {
const index: Index<any> = {
name: ele[0].Key_name,
attributes: ele.map(ele2 => ({
name: ele2.Column_name,
direction: ele2.Collation === 'D' ? 'DESC' : (ele2.Collation === 'A' ? 'ASC' : undefined),
size: ele2.Sub_part || undefined,
})),
};
if (ele[0].Non_unique === 0 || ele[0].Index_type.toUpperCase() !== 'BTREE') {
index.config = {};
if (ele[0].Non_unique === 0) {
index.config.unique = true;
}
if (ele[0].Index_type.toUpperCase() !== 'BTREE') {
index.config.type = ele[0].Index_type.toLowerCase() as any;
}
}
return index;
}
);
Object.assign(result[table], {
indexes,
});
}
}
return result;
}
}

27
src/sqlTranslator.ts
View File

@ -4,7 +4,8 @@ import { assign, cloneDeep, difference, identity, intersection, keys, set } from
import {
Attribute, EntityDict, EXPRESSION_PREFIX, Index, OperateOption,
Q_FullTextValue, Ref, RefOrExpression, SelectOption, StorageSchema, SubQueryPredicateMetadata,
TriggerDataAttribute, CreateAtAttribute, UpdateAtAttribute, DeleteAtAttribute, SeqAttribute, TriggerUuidAttribute
TriggerDataAttribute, CreateAtAttribute, UpdateAtAttribute, DeleteAtAttribute, SeqAttribute, TriggerUuidAttribute,
PrimaryKeyAttribute
} from "oak-domain/lib/types";
import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
import { DataType } from "oak-domain/lib/types/schema/DataTypes";
@ -29,7 +30,7 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
const { attributes, indexes } = schema[entity];
// 增加默认的属性
assign(attributes, {
id: {
[PrimaryKeyAttribute]: {
type: 'char',
params: {
length: 36,
@ -78,7 +79,7 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
name: DeleteAtAttribute,
}],
}, {
name: `${entity}_trigger_uuid`,
name: `${entity}_trigger_uuid_auto_create`,
attributes: [{
name: TriggerUuidAttribute,
}]
@ -159,6 +160,18 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
}
if (indexes) {
for (const index of indexes) {
const { attributes, config } = index;
if (!config?.type || config.type === 'btree') {
if (!attributes.find(
(ele) => ele.name === DeleteAtAttribute
)) {
attributes.push({
name: DeleteAtAttribute,
});
}
}
}
indexes.push(...intrinsticIndexes);
}
else {
@ -1235,4 +1248,12 @@ export abstract class SqlTranslator<ED extends EntityDict & BaseEntityDict> {
const result = SqlString.escape(value);
return result;
}
/**比较两段sql是否完全一致，这里是把所有的空格去掉了 */
compareSql(sql1: string, sql2: string) {
const reg = /[\t\r\f\n\s]/g;
return sql1.replaceAll(reg, '') === sql2.replaceAll(reg, '');
}
}

31
test/testMySQLStore.ts
View File

@ -1735,6 +1735,24 @@ describe('test mysqlstore', function () {
}
}
}, context, {});
assert (typeof (row[0]?.data as any).price === 'string');
// todo 这个暂时还没搞定。需求也不是太强
const row1 = await store.select('oper', {
data: {
id: 1,
data: {
name: 1,
price: 2,
},
},
filter: {
data: {
price: [undefined, 400],
}
}
}, context, {});
// assert (typeof (row[0]?.data as any).price === 'object');
const row2 = await store.select('oper', {
data: {
@ -2223,6 +2241,19 @@ describe('test mysqlstore', function () {
assert(r8.map(ele => ele.id).includes(id3));
});
it('[2.0.1]read schema', async () => {
const result = await store.readSchema();
console.log(result);
});
it('[2.0.2]diff schema', async () => {
const result = await store.makeUpgradePlan();
console.log(result);
});
after(() => {
store.disconnect();
});