add elasticsearch

This commit is contained in:
COOL 2025-04-02 13:02:01 +08:00
parent 69536b0da1
commit 79e49fea73
18 changed files with 9186 additions and 0 deletions

11
es/.editorconfig Normal file
View File

@ -0,0 +1,11 @@
# 🎨 editorconfig.org
root = true
[*]
charset = utf-8
end_of_line = lf
indent_style = space
indent_size = 2
trim_trailing_whitespace = true
insert_final_newline = true

22
es/.eslintrc.json Normal file
View File

@ -0,0 +1,22 @@
{
"extends": "./node_modules/mwts/",
"ignorePatterns": ["node_modules", "dist", "test", "jest.config.js", "typings"],
"env": {
"jest": true
},
"rules": {
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/ban-ts-comment": "off",
"node/no-extraneous-import": "off",
"@typescript-eslint/no-this-alias": "off",
"no-empty": "off",
"node/no-extraneous-require": "off",
"node/no-unpublished-import": "off",
"eqeqeq": "off",
"node/no-unsupported-features/node-builtins": "off",
"@typescript-eslint/ban-types": "off",
"no-control-regex": "off",
"prefer-const": "off"
}
}

4
es/.gitattributes vendored Normal file
View File

@ -0,0 +1,4 @@
*.js text eol=lf
*.json text eol=lf
*.ts text eol=lf
*.code-snippets text eol=lf

13
es/.gitignore vendored Normal file
View File

@ -0,0 +1,13 @@
logs/
npm-debug.log
yarn-error.log
node_modules/
coverage/
dist/
.idea/
run/
.DS_Store
*.sw*
*.un~
.tsbuildinfo
.tsbuildinfo.*

3
es/.prettierrc.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = {
...require('mwts/.prettierrc.json')
}

13
es/README.md Normal file
View File

@ -0,0 +1,13 @@
# Cool Admin Node
cool-admin一个很酷的后台权限管理系统开源免费模块化、插件化、极速开发CRUD方便快速构建迭代后台管理系统支持serverless、docker、普通服务器等多种方式部署 到 官网 进一步了解。
[https://cool-js.com](https://cool-js.com)
## 特性
- 🔥 **AI 编码** - 从页面到后端代码,部分功能实现零代码
- 🎯 **Ai 流程编排** - 专门为 AI 开发设计,几乎无需编码,拖拽即可
- 🎨 **扩展插件** - 可插拔式的插件机制,支付、短信等功能可通过后台动态安装卸载
- 📦 **代码简洁** - 不同于一般代码生成器生成冗余代码,只需极少编码即可实现大部分需求
- 🚀 **微服务支持** - 基于 Moleculer 的微服务架构,支持分布式部署

10
es/index.d.ts vendored Normal file
View File

@ -0,0 +1,10 @@
export * from './dist/index';
declare module '@midwayjs/core/dist/interface' {
interface MidwayConfig {
book?: PowerPartial<{
a: number;
b: string;
}>;
}
}

7
es/jest.config.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testPathIgnorePatterns: ['<rootDir>/test/fixtures'],
coveragePathIgnorePatterns: ['<rootDir>/test/'],
setupFilesAfterEnv: ['./jest.setup.js']
};

1
es/jest.setup.js Normal file
View File

@ -0,0 +1 @@
jest.setTimeout(30000);

56
es/package.json Normal file
View File

@ -0,0 +1,56 @@
{
"name": "@cool-midway/es",
"version": "8.0.0",
"description": "cool-js.com elasticsearch",
"main": "dist/index.js",
"typings": "index.d.ts",
"scripts": {
"build": "mwtsc --cleanOutDir",
"test": "cross-env NODE_ENV=unittest jest",
"cov": "jest --coverage",
"lint": "mwts check",
"lint:fix": "mwts fix"
},
"keywords": [
"cool",
"cool-admin",
"cooljs"
],
"author": "COOL",
"files": [
"dist/**/*.js",
"dist/**/*.d.ts",
"index.d.ts"
],
"readme": "README.md",
"license": "MIT",
"repository": {
"type": "git",
"url": "https://cool-js.com"
},
"devDependencies": {
"@cool-midway/core": "8.0.1",
"@midwayjs/core": "^3.20.0",
"@midwayjs/koa": "^3.20.0",
"@midwayjs/logger": "^3.4.2",
"@midwayjs/mock": "^3.20.0",
"@midwayjs/redis": "^3.20.0",
"@midwayjs/typeorm": "^3.20.0",
"@types/jest": "^29.5.14",
"@types/node": "^22.10.7",
"cross-env": "^7.0.3",
"jest": "^29.7.0",
"lodash": "^4.17.21",
"moment": "^2.30.1",
"mwts": "^1.3.0",
"mwtsc": "^1.15.1",
"sqlstring": "^2.3.3",
"ts-jest": "^29.2.5",
"typeorm": "^0.3.20",
"typescript": "^5.7.3",
"uuid": "^11.0.5"
},
"dependencies": {
"@elastic/elasticsearch": "^8.1.0"
}
}

8161
es/pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

615
es/src/base.ts Normal file
View File

@ -0,0 +1,615 @@
import { CoolEventManager } from '@cool-midway/core';
import { Client } from '@elastic/elasticsearch';
import { WaitForActiveShards } from '@elastic/elasticsearch/lib/api/types';
import { Inject, Logger } from '@midwayjs/core';
import { ILogger } from '@midwayjs/logger';
import { EsConfig } from '.';
/**
* Es索引基类
*/
export class BaseEsIndex {
// 索引
public index: string;
// es客户端
public client: Client;
// 日志
@Logger()
coreLogger: ILogger;
// 事件
@Inject('cool:coolEventManager')
coolEventManager: CoolEventManager;
/**
*
* @param index
*/
setIndex(index: string) {
this.index = index;
}
/**
* es数据变更事件
* @param method
* @param data
*/
async handleDataChange(index, method, data) {
this.index = index;
const {
id,
ids,
bodys,
body,
type,
refresh,
waitForActiveShards,
properties,
config,
} = data;
switch (method) {
case 'upsert':
await this.upsert(body, refresh, waitForActiveShards);
break;
case 'batchIndex':
await this.batchIndex(bodys, type, refresh, waitForActiveShards);
break;
case 'deleteById':
await this.deleteById(id, refresh, waitForActiveShards);
break;
case 'deleteByIds':
await this.deleteByIds(ids, refresh, waitForActiveShards);
break;
case 'deleteByQuery':
await this.deleteByQuery(body, refresh, waitForActiveShards);
break;
case 'updateById':
await this.updateById(body, refresh, waitForActiveShards);
break;
case 'updateByQuery':
await this.updateByQuery(body, refresh, waitForActiveShards);
break;
case 'createIndex':
await this.updateByQuery(properties, config);
break;
}
}
/**
*
* @param method
* @param data
*/
async esDataChange(method, data) {
this.coolEventManager?.emit('esDataChange', this.index, method, data);
}
/**
*
* @param client
*/
setClient(client: Client) {
this.client = client;
}
/**
* body
* @param condition
*/
async objToBody(condition: any) {
const body = {
query: {
bool: {
must: [],
},
},
};
for (const key in condition) {
body.query.bool.must.push({
term: {
[key]: condition[key],
},
});
}
return body;
}
/**
*
* @param condition
* @param size
* @returns
*/
async findBy(condition: any, size?: number) {
const body = await this.objToBody(condition);
return this.find(body, size);
}
/**
*
* @param condition
* @param page
* @param size
*/
async findPageBy(condition: any, page?: number, size?: number) {
const body = await this.objToBody(condition);
return this.findPage(body, page, size);
}
/**
*
* @param body
*/
async find(body?: any, size?: number) {
if (!body) {
body = {};
}
if (!body.size) {
body.size = size ? size : 10000;
}
return this.client
.search({
index: this.index,
body,
})
.then(res => {
return (
res.hits.hits.map(e => {
e._source['id'] = e._id;
const _source: any = e._source;
['_id', '_index', '_score', '_source'].forEach(key => {
delete e[key];
});
return {
..._source,
...e,
};
}) || []
);
});
}
/**
*
* @param body
* @param page
* @param size
*/
async findPage(body?: any, page?: number, size?: number) {
if (!page) {
page = 1;
}
if (!body) {
body = {};
}
if (!size && !body.size) {
size = 20;
body.size = size;
}
const total = await this.findCount(body);
body.from = (page - 1) * size;
return this.client.search({ index: this.index, body }).then(res => {
const result =
res.hits.hits.map(e => {
e._source['id'] = e._id;
const _source: any = e._source;
['_id', '_index', '_score', '_source'].forEach(key => {
delete e[key];
});
return {
..._source,
...e,
};
}) || [];
return {
list: result,
pagination: {
page,
size,
total,
},
};
});
}
/**
* ID查询
* @param id
* @returns
*/
async findById(id) {
return this.client
.get({
index: this.index,
id,
})
.then(res => {
res._source['id'] = res._id;
return res._source || undefined;
})
.catch(e => {
return undefined;
});
}
/**
* ID查询
* @param ids
* @returns
*/
async findByIds(ids: string[]) {
return this.client
.mget({ index: this.index, body: { ids } })
.then(res => {
const result = res.docs.map((e: any) => {
e._source.id = e._id;
return e._source || 'undefined';
});
return result.filter(e => {
return e !== 'undefined';
});
})
.catch(e => {
return undefined;
});
}
/**
*
* @param body
* @param refresh
* @param waitForActiveShards
* @returns
*/
async upsert(
body: any,
refresh?: boolean | 'wait_for',
waitForActiveShards?: WaitForActiveShards
) {
if (refresh == undefined) {
refresh = true;
}
if (body.id) {
this.esDataChange('upsert', {
body,
refresh,
waitForActiveShards,
});
const id = body.id;
delete body.id;
return this.client.index({
id,
index: this.index,
wait_for_active_shards: waitForActiveShards,
refresh,
body,
});
} else {
return this.client
.index({
index: this.index,
wait_for_active_shards: waitForActiveShards,
refresh,
body,
})
.then(res => {
this.esDataChange('upsert', {
body: {
...body,
id: res._id,
},
refresh,
waitForActiveShards,
});
return res;
});
}
}
/**
*
* @param bodys
* @param type
* @param refresh
* @param waitForActiveShards
* @returns
*/
async batchIndex(
bodys: any[],
type: 'index' | 'create' | 'delete' | 'update',
refresh?: boolean | 'wait_for',
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('batchIndex', {
bodys,
type,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
const list = [];
for (const body of bodys) {
const typeO = {};
typeO[type] = { _index: this.index, _id: body.id };
if (body.id) {
delete body.id;
}
list.push(typeO);
if (type !== 'delete') {
if (type == 'update') {
list.push({ doc: body });
} else {
list.push(body);
}
}
}
return this.client.bulk({
wait_for_active_shards: waitForActiveShards,
index: this.index,
refresh,
body: list,
});
}
/**
*
* @param id
* @param refresh
* @param waitForActiveShards
* @returns
*/
async deleteById(
id,
refresh?: boolean | 'wait_for',
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('deleteById', {
id,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
try {
return this.client.delete({
index: this.index,
refresh,
wait_for_active_shards: waitForActiveShards,
id,
});
} catch {}
}
/**
*
* @param ids
* @param refresh
* @param waitForActiveShards
* @returns
*/
async deleteByIds(
ids: string[],
refresh?: boolean,
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('deleteByIds', {
ids,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
const body = {
query: {
bool: {
must: [
{
terms: {
_id: ids,
},
},
],
},
},
};
return this.client.deleteByQuery({
index: this.index,
refresh,
wait_for_active_shards: waitForActiveShards,
body,
});
}
/**
*
* @param body
* @param refresh
* @param waitForActiveShards
* @returns
*/
async deleteByQuery(
body,
refresh?: boolean,
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('deleteByQuery', {
body,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
return this.client.deleteByQuery({
index: this.index,
refresh,
wait_for_active_shards: waitForActiveShards,
body,
});
}
/**
*
* @param body
* @param refresh
* @param waitForActiveShards
* @returns
*/
async updateById(
body,
refresh?: boolean | 'wait_for',
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('updateById', {
body,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
const id = body.id;
delete body.id;
return this.client.update({
wait_for_active_shards: waitForActiveShards,
index: this.index,
id: id,
refresh,
body: {
doc: body,
},
});
}
/**
*
* @param body
* @param refresh
* @param waitForActiveShards
*/
async updateByQuery(
body,
refresh?: boolean,
waitForActiveShards?: WaitForActiveShards
) {
this.esDataChange('updateByQuery', {
body,
refresh,
waitForActiveShards,
});
if (refresh == undefined) {
refresh = true;
}
return this.client.updateByQuery({
index: this.index,
refresh,
wait_for_active_shards: waitForActiveShards,
body,
});
}
/**
*
* @param body
*/
async findCount(body?: any) {
let _body = Object.assign({}, body || {});
delete _body.from;
delete _body.size;
delete _body.sort;
return this.client
.count({
index: this.index,
body: _body,
})
.then(res => {
return res.count;
})
.catch(e => {
return undefined;
});
}
/**
*
* @param config
*/
async createIndex(
properties: {},
config: EsConfig = {
name: '',
replicas: 1,
shards: 8,
analyzers: [],
}
) {
this.esDataChange('createIndex', {
properties,
config,
});
const body = {
settings: {
number_of_shards: config.shards,
number_of_replicas: config.replicas,
analysis: {
analyzer: {
comma: { type: 'pattern', pattern: ',' },
blank: { type: 'pattern', pattern: ' ' },
},
},
mapping: {
nested_fields: {
limit: 100,
},
},
},
mappings: {
properties: {},
},
};
if (config.analyzers) {
for (const analyzer of config.analyzers) {
for (const key in analyzer) {
body.settings.analysis.analyzer[key] = analyzer[key];
}
}
}
const param = {
index: this.index,
body,
};
param.body = body;
param.body.mappings.properties = properties;
this.client.indices.exists({ index: this.index }).then(async res => {
if (!res) {
await this.client.indices.create(param).then(res => {
if (res.acknowledged) {
console.info(
'\x1B[36m [cool:core] midwayjs cool elasticsearch ES索引创建成功: ' +
this.index +
' \x1B[0m'
);
}
});
} else {
const updateParam = {
index: this.index,
body: param.body.mappings,
};
await this.client.indices.putMapping(updateParam).then(res => {
if (res.acknowledged) {
console.info(
'\x1B[36m [cool:core] midwayjs cool elasticsearch ES索引更新成功: ' +
this.index +
' \x1B[0m'
);
}
});
}
});
}
}

View File

@ -0,0 +1,4 @@
export const customKey = {
a: 1,
b: 'hello',
};

19
es/src/configuration.ts Normal file
View File

@ -0,0 +1,19 @@
import { Configuration } from '@midwayjs/core';
import * as DefaultConfig from './config/config.default';
import { IMidwayContainer } from '@midwayjs/core';
import { CoolElasticSearch } from './elasticsearch';
@Configuration({
namespace: 'cool:es',
importConfigs: [
{
default: DefaultConfig,
},
],
})
export class CoolEsConfiguration {
async onReady(container: IMidwayContainer) {
await container.getAsync(CoolElasticSearch);
// TODO something
}
}

View File

@ -0,0 +1,41 @@
import {
Scope,
ScopeEnum,
saveClassMetadata,
saveModule,
} from '@midwayjs/core';
export const COOL_ES_KEY = 'decorator:cool:es';
export interface EsConfig {
shards?: number;
name: string;
replicas?: number;
analyzers?: any[];
}
/**
*
* @param config
* @returns
*/
export function CoolEsIndex(
config: EsConfig | string = {
name: '',
replicas: 1,
shards: 8,
analyzers: [],
}
): ClassDecorator {
if (typeof config == 'string') {
config = { name: config, replicas: 1, shards: 8, analyzers: [] };
}
return (target: any) => {
// 将装饰的类,绑定到该装饰器,用于后续能获取到 class
saveModule(COOL_ES_KEY, target);
// 保存一些元数据信息,任意你希望存的东西
saveClassMetadata(COOL_ES_KEY, config, target);
// 指定 IoC 容器创建实例的作用域,这里注册为请求作用域,这样能取到 ctx
Scope(ScopeEnum.Singleton)(target);
};
}

160
es/src/elasticsearch.ts Normal file
View File

@ -0,0 +1,160 @@
import {
Provide,
getClassMetadata,
App,
Logger,
Inject,
Init,
Scope,
ScopeEnum,
Config,
listModule,
} from '@midwayjs/core';
import { COOL_ES_KEY, EsConfig } from './decorator/elasticsearch';
import { IMidwayApplication } from '@midwayjs/core';
import { CoolCoreException, CoolEventManager } from '@cool-midway/core';
import { ILogger } from '@midwayjs/logger';
import { Client } from '@elastic/elasticsearch';
import * as _ from 'lodash';
import { CoolEsConfig, ICoolEs } from '.';
/**
*
*/
@Provide()
@Scope(ScopeEnum.Singleton)
export class CoolElasticSearch {
@App()
app: IMidwayApplication;
@Logger()
coreLogger: ILogger;
@Config('cool.es')
esConfig: CoolEsConfig;
client: Client;
@Inject('cool:coolEventManager')
coolEventManager: CoolEventManager;
@Init()
async init() {
if (!this.esConfig?.nodes) {
throw new CoolCoreException('es.nodes config is require');
}
if (this.esConfig.nodes.length == 1) {
this.client = new Client({
node: this.esConfig.nodes[0],
...this.esConfig.options,
});
} else {
this.client = new Client({
nodes: this.esConfig.nodes,
...this.esConfig.options,
});
}
this.client.ping({}, { requestTimeout: 30000 }).then(res => {
if (res) {
this.coolEventManager.emit('esReady', this.client);
this.scan();
}
});
}
async scan() {
const modules = listModule(COOL_ES_KEY);
for (let module of modules) {
const cls: ICoolEs = await this.app
.getApplicationContext()
.getAsync(module);
const data = getClassMetadata(COOL_ES_KEY, module);
this.createIndex(cls, data);
}
}
/**
*
* @param method
* @param data
*/
async esDataChange(method, data) {
//this.coolEventManager.emit('esDataChange', { method, data });
}
/**
*
* @param cls
* @param config
*/
async createIndex(cls, config: EsConfig) {
cls.index = config.name;
cls.client = this.client;
const body = {
settings: {
number_of_shards: config.shards,
number_of_replicas: config.replicas,
analysis: {
analyzer: {
comma: { type: 'pattern', pattern: ',' },
blank: { type: 'pattern', pattern: ' ' },
},
},
mapping: {
nested_fields: {
limit: 100,
},
},
},
mappings: {
properties: {},
},
};
if (config.analyzers) {
for (const analyzer of config.analyzers) {
for (const key in analyzer) {
body.settings.analysis.analyzer[key] = analyzer[key];
}
}
}
const param = {
index: config.name,
body,
};
param.body = body;
param.body.mappings.properties = cls.indexInfo();
this.esDataChange('createIndex', {
properties: param.body.mappings.properties,
config,
});
this.client.indices.exists({ index: config.name }).then(async res => {
if (!res) {
await this.client.indices.create(param).then(res => {
if (res.acknowledged) {
this.coreLogger.info(
'\x1B[36m [cool:core] midwayjs cool elasticsearch ES索引创建成功: ' +
config.name +
' \x1B[0m'
);
}
});
} else {
const updateParam = {
index: config.name,
body: param.body.mappings,
};
await this.client.indices.putMapping(updateParam).then(res => {
if (res.acknowledged) {
this.coreLogger.info(
'\x1B[36m [cool:core] midwayjs cool elasticsearch ES索引更新成功: ' +
config.name +
' \x1B[0m'
);
}
});
}
});
}
}

18
es/src/index.ts Normal file
View File

@ -0,0 +1,18 @@
import { ClientOptions } from '@elastic/elasticsearch';
export { CoolEsConfiguration as Configuration } from './configuration';
export * from './elasticsearch';
export * from './decorator/elasticsearch';
export * from './base';
export interface ICoolEs {
indexInfo(): Object;
}
export interface CoolEsConfig {
nodes: string[];
options?: ClientOptions;
}

28
es/tsconfig.json Normal file
View File

@ -0,0 +1,28 @@
{
"compileOnSave": true,
"compilerOptions": {
"target": "es2018",
"module": "commonjs",
"moduleResolution": "node",
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"inlineSourceMap":false,
"noImplicitThis": true,
"noUnusedLocals": true,
"stripInternal": true,
"skipLibCheck": true,
"noImplicitReturns": false,
"pretty": true,
"declaration": true,
"forceConsistentCasingInFileNames": true,
"outDir": "dist",
"rootDir": "src"
},
"exclude": [
"*.js",
"*.ts",
"dist",
"node_modules",
"test"
]
}