cube.js 支持一种基于package.json deps 模式的schema 发现，但是目前默认是没有开启的，今天在开发基于
s3扩展的时候觉得也有必要支持下，所以基于es6 的默认函数参数模式，将 async dataSchemaFiles(includeDependencies=true)
这样默认就可以集成基于npm包的schema 发现了
npm包约定
包的名称需要是以-schema 结尾的，同时加载的是.js 的文件（支持递归）
参考s3支持的扩展代码
参考了官方的FileRepository 对于npm 包的处理
const Minio = require('minio')
const streamToPromise = require('stream-to-promise');
const path = require('path');
const fs = require('fs-extra');
const R = require('ramda');
/**
* mys3 file repository
*/
class S3FileRepository {
// init config with env
constructor() {
this.minioClient = new Minio.Client({
endPoint: process.env.cube_s3_endpoint,
port: parseInt(process.env.cube_s3_port),
useSSL: process.env.cube_s3_ssl == "true" ? true : false,
accessKey: process.env.cube_s3_accesskey,
secretKey: process.env.cube_s3_secretkey
});
}
async dataSchemaFiles(includeDependencies=true) {
var localminioClient = this.minioClient
var bucket = process.env.cube_s3_bucket
var Files = await streamToPromise(localminioClient.listObjectsV2(bucket, "", true))
var fileContents = []
for (const file of Files) {
try {
const fileBuffer = await streamToPromise(await localminioClient.getObject(bucket, file.name))
let fileItemContent = fileBuffer.toString('utf-8');
fileContents.push({ fileName: file.name, content: fileItemContent })
}
catch (e) {
console.log(e)
}
}
if (includeDependencies) {
fileContents = fileContents.concat(await this.readModules());
}
return fileContents;
}
async readModules() {
const packageJson = JSON.parse(await fs.readFile('package.json', 'utf-8'));
const files = await Promise.all(
Object.keys(packageJson.dependencies).map(async module => {
if (R.endsWith('-schema', module)) {
return this.readModuleFiles(path.join('node_modules', module));
}
return [];
})
);
return files.reduce((a, b) => a.concat(b));
}
async readModuleFiles(modulePath) {
const files = await fs.readdir(modulePath);
return (await Promise.all(
files.map(async file => {
const fileName = path.join(modulePath, file);
const stats = await fs.lstat(fileName);
if (stats.isDirectory()) {
return this.readModuleFiles(fileName);
} else if (R.endsWith('.js', file)) {
const content = await fs.readFile(fileName, 'utf-8');
return [
{
fileName,
content,
readOnly: true
}
];
} else {
return [];
}
})
)).reduce((a, b) => a.concat(b), []);
}
}
module.exports = {
repositoryFactory: ({ authInfo }) => new S3FileRepository(),
};
参考使用
还是以前的参考代码
- 创建一个约定的schema npm 包
demo 已经push npm 官方仓库了，可以直接使用
mydemo.js
cube(`dalongrong`, {
sql: `SELECT * FROM public.demoapp`,
joins: {
},
preAggregations: {
mydemo: {
type: `rollup`,
measureReferences: [Demoapp.count],
dimensionReferences: [name],
external: true
}
},
measures: {
count: {
type: `count`,
drillMembers: [name, id]
},
"price": {
sql: "id",
"type":"count"
}
},
dimensions: {
name: {
sql: `name`,
type: `string`
},
id: {
sql: `id`,
type: `number`,
primaryKey: true
}
}
});
package.json
{
"name": "@dalongrong/mycubejs-schema",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"publishConfig": {
"registry": "https://registry.npmjs.org/"
},
"scripts": {
"p": "yarn publish --access public"
}
}
- 集成s3 FileRepository
具体s3配置部分和以前的一样，主要是添加一个符合schema发现约定的npm包到deps 就可以了
参考项目的package.json
{
"name": "pre-age",
"version": "0.0.1",
"private": true,
"scripts": {
"dev": "./node_modules/.bin/cubejs-server",
"app": "node index.js"
},
"devDependencies": {
"@cubejs-backend/cubestore-driver": "^0.25.24",
"@cubejs-backend/postgres-driver": "^0.25.24",
"@cubejs-backend/server": "^0.25.24"
},
"dependencies": {
"@dalongrong/cube-s3repository": "^1.0.1",
"@dalongrong/mycubejs-schema": "^1.0.0", // 自己开发的符合schema 的npm 包，demo 以及push npm 官方仓库
"node-fetch": "^2.6.1",
"throng": "^5.0.0"
}
}
- 效果
基于npm schema 发现的定义 
说明
关于支持此特性的S3FileRepository 已经push npm 官方仓库了，大家可以直接使用
参考资料
https://cube.dev/docs/config#schema-file-repository
https://github.com/rongfengliang/cube.js-s3-filerepository
https://www.npmjs.com/package/@dalongrong/mycubejs-schema
https://www.npmjs.com/package/@dalongrong/cube-s3repository