一，step1页面
<template>
<div
class="add-data card"
v-loading="loading"
:element-loading-text="loadingText"
>
<div class="add-data-box">
<div class="steps">
<div
class="step"
:class="currentComponent === AddForm ? 'action' : 'success'"
>
<div class="step-icon">1</div>
<div class="success-icon"><i class="el-icon-check" /></div>
<div class="step-name">配置数据信息</div>
</div>
<template>
<div class="step-line-box">
<i class="step-line" />
</div>
<div class="step" :class="{ action: currentComponent !== AddForm }">
<div class="step-icon">2</div>
<div class="step-name">元数据管理</div>
</div>
</template>
</div>
<keep-alive>
<component
:is="currentComponent"
ref="currentComponent"
:set-loading="setLoading"
:fields="fields"
@next="handleNext"
@handleTypeChange="handleTypeChange"
/>
</keep-alive>
<div class="text-center mt-3" v-if="currentComponent === ConfigData">
<el-button @click="back">上一步</el-button>
<el-button type="primary" @click="ok">完成</el-button>
</div>
</div>
</div>
</template>
<script>
import AddForm from './AddForm'
import ConfigData from './ConfigData'
import { createMySql, findFiledsList, checkDataSetStatus } from '@/api/dataSet'
import CheckData from './CheckData'
export default {
components: { AddForm, ConfigData },
props: {},
data() {
return {
currentComponent: AddForm,
loading: false,
AddForm,
ConfigData,
formData: null,
fields: [],
dialogVisible: true,
isCsvComp: true,
loadingText: '',
sampleId: '',
}
},
created() {
this.setLoading = this.setLoading.bind(this)
},
computed: {},
methods: {
handleTypeChange(comp) {
this.isCsvComp = comp
},
setLoading(isLoading, loadingText = '检测中') {
this.loading = isLoading
this.loadingText = loadingText
},
handleNext(formAndParams) {
if (formAndParams === null) return
this.formData = formAndParams.form
this.setLoading(true, '')
createMySql(formAndParams.form)
.then(async res => {
this.sampleId = res.businessId
const result = await findFiledsList({ sampleId: res.businessId })
this.fields = result.sampleFields
this.currentComponent = ConfigData
})
.finally(() => {
this.setLoading(false, '')
})
},
back() {
this.currentComponent = AddForm
this.fields = []
this.formData = null
this.sampleId = ''
},
async ok() {
this.setLoading(true, '保存中')
const { fields } = this
const checkResult = new CheckData(this.formData, fields).checkAll()
if (!checkResult.status) {
this.$message.warning(checkResult.list[0].message)
this.setLoading(false)
return
}
checkDataSetStatus({
type: '1',
sampleId: this.sampleId,
sampleFields: fields,
})
.then(() => {
this.$message.success('添加成功')
this.setLoading(false)
this.$store.commit('DEL_VIEW_TAG', {
view: this.$route,
callback: () => {
this.$router.push({ name: 'DataList' })
},
})
})
.catch(err => {
this.setLoading(false)
this.$message.error(err.respMsg)
})
.finally(() => {})
},
},
}
</script>
<style lang="scss" scoped>
.add-data {
padding: 40px 20px 24px 20px;
margin-bottom: 24px;
min-height: calc(100% - 88px);
}
.steps,
.step {
display: flex;
}
.step-name {
font-size: 14px;
}
/deep/ .el-form-item__label,
/deep/ .el-input,
/deep/ .el-textarea__inner {
font-size: 12px;
}
.font-twelve {
/deep/ .el-select-dropdown__item {
font-size: 12px;
}
}
.steps {
justify-content: center;
height: 34px;
padding-bottom: 32px;
}
.step {
& > div {
display: flex;
align-items: center;
justify-content: center;
}
&-icon {
32px;
height: 32px;
border-radius: 50%;
border: 1px solid rgba(0, 0, 0, 0.15);
color: rgba(0, 0, 0, 0.25);
}
&-name {
color: rgba(0, 0, 0, 0.45);
}
&.action &-icon {
background: #e95d21;
border-color: #e95d21;
color: #fff;
}
&.action &-name {
color: rgba(0, 0, 0, 0.85);
}
.success-icon {
display: none;
}
&.success .success-icon {
display: flex;
32px;
height: 32px;
border: #e95d21 solid 1px;
border-radius: 50%;
color: #e95d21;
}
&.success &-icon {
display: none;
}
&-name {
margin-left: 6px;
}
&-line-box {
160px;
margin: 0 32px;
display: flex;
align-items: center;
}
&-line {
100%;
height: 1px;
background: rgba(0, 0, 0, 0.15);
}
}
</style>
二，step1页面中AddForm组件
<template>
<div style="max- 600px; margin: auto">
<!-- 输入框组件 -->
<BaseForm
ref="baseForm"
@typeChange="handleTypeChange"
@sampleTypeChange="value => (sampleType = value)"
/>
<!-- 文件上传组件 -->
<keep-alive>
<component
:is="MysqlOrCsvComponent"
ref="MysqlOrCsvComponent"
:sample-type="sampleType"
:set-loading="setLoading"
/>
</keep-alive>
<div slot="footer" class="dialog-footer text-center">
<el-button @click="cancel">取消</el-button>
<el-button type="primary" @click="next">下一步</el-button>
</div>
</div>
</template>
<script>
import BaseForm from './BaseForm'
import CsvTypeForm from './CsvTypeForm'
import MySqlTypeForm from './MySqlTypeForm'
export default {
components: { BaseForm, CsvTypeForm, MySqlTypeForm },
props: {
setLoading: { type: Function, required: true },
},
data() {
return {
MysqlOrCsvComponent: CsvTypeForm,
baseForm: null,
sampleType: '1',
}
},
methods: {
cancel() {
this.$store.commit('DEL_VIEW_TAG', {
view: this.$route,
callback: nearTag => {
if (nearTag.last) {
this.$router.push(nearTag.last.fullPath)
} else if (nearTag.next) {
this.$router.push(nearTag.next.fullPath)
} else {
this.$router.push('/')
}
},
})
},
async next() {
const baseForm = this.$refs.baseForm.getForm()
const csvOrMysqlForm = this.$refs.MysqlOrCsvComponent
? this.$refs.MysqlOrCsvComponent.getForm()
: null
if (baseForm === null || csvOrMysqlForm === null) return
let params = null
if (this.MysqlOrCsvComponent === CsvTypeForm) {
params = { type: '1', path: csvOrMysqlForm.path }
}
if (this.MysqlOrCsvComponent === MySqlTypeForm) {
params = { ...csvOrMysqlForm, type: '2' }
}
this.$emit('next', {
form: { ...baseForm, ...csvOrMysqlForm },
params,
})
},
handleTypeChange(value) {
if (value === '1') {
this.MysqlOrCsvComponent = CsvTypeForm
this.$emit('handleTypeChange', true)
} else if (value === '3') {
this.MysqlOrCsvComponent = MySqlTypeForm
this.$emit('handleTypeChange', false)
}
},
},
}
</script>
三，step1页面中AddForm组件中的BaseForm
<template>
<el-form :model="form" label-width="100px" :rules="rules" ref="form">
<el-form-item label="数据集名称" prop="name">
<el-input v-model="form.name" placeholder="请输入数据集名称" />
</el-form-item>
<el-form-item label="数据集简介" prop="comments">
<el-input
v-model="form.comments"
type="textarea"
maxlength="200"
show-word-limit
:autosize="{ minRows: 2, maxRows: 15 }"
placeholder="请输入数据集简介"
/>
</el-form-item>
<el-form-item label="数据集类型" prop="scope">
<el-radio
v-for="item of scopeOptions"
:key="item.value"
:label="item.value"
v-model="form.scope"
>{{ item.label }}</el-radio
>
</el-form-item>
<el-form-item label="样本集类别" prop="scope">
<el-radio
:disabled="form.scope === '2'"
v-for="item of sampleTypeOptions"
:key="item.value"
:label="item.value"
v-model="form.sampleType"
>{{ item.label }}</el-radio
>
</el-form-item>
<el-form-item label="数据源类型" prop="type">
<el-select
popper-class="zindex-calss font-twelve"
v-model="form.type"
@change="typeChange"
>
<el-option
v-for="item of typeOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
</el-form>
</template>
<script>
import { formNameRule } from '@/utils/validate/formRule'
const createForm = () => ({
name: '',
comments: '',
scope: '1',
type: '1',
sampleType: '1',
})
export default {
data() {
return {
form: createForm(),
rules: {
name: [
{ required: true, message: '请输入数据集名称', trigger: 'blur' },
...formNameRule,
],
comments: [
{ required: false, message: '请输入数据集简介', trigger: 'blur' },
],
scope: [
{ required: true, message: '请选择数据集类型', trigger: 'change' },
],
type: [
{ required: true, message: '请选择数据源类型', trigger: 'change' },
],
sampleType: [
{ required: true, message: '请选择样本集类别', trigger: 'change' },
],
},
typeOptions: [
{ label: 'CSV', value: '1' },
{ label: 'MySql', value: '3' },
],
scopeOptions: [
{ label: '训练数据集', value: '1' },
{ label: '预测数据集', value: '2' },
],
sampleTypeOptions: [
{ label: 'Y样本集', value: '2' },
{ label: 'X样本集', value: '1' },
],
}
},
watch: {
'form.scope'(value) {
if (value === '2') {
this.form.sampleType = '1'
}
},
'form.sampleType'(value) {
this.$emit('sampleTypeChange', value)
},
},
methods: {
getForm() {
let formValid
this.$refs.form.validate(valid => {
formValid = valid
})
if (formValid) {
return { ...this.form }
}
return null
},
typeChange(value) {
this.$emit('typeChange', value)
},
cancel() {
this.form = createForm()
},
},
}
</script>
四，step1页面中AddForm组件中的CsvTypeForm
<template>
<div>
<el-form :model="form" label-width="100px" ref="form">
<el-form-item
label="上传文件"
prop="path"
:rules="[{ required: true, message: '请上传文件', trigger: 'change' }]"
>
<el-upload
ref="upload"
class="upload"
action=""
drag
:on-remove="handleRemove"
:limit="1"
:multiple="false"
:on-exceed="handleExceed"
:on-change="onFileChange"
:auto-upload="false"
>
<i class="el-icon-receiving mt-3" style="font-size: 67px" />
<div class="el-upload__text mt-1 mb-1">
将文件拖到此处，或<em>点击上传</em>
</div>
<div
class="text-center"
style="font-size: 12px; color: #ccc; line-height: 20px"
>
文件仅支持csv格式，最大限制1G
</div>
<div
slot="tip"
class="text-right"
style=" 360px; font-size: 12px"
>
<span>下载模板: </span>
<el-button type="text" @click="download()">样本集示例</el-button>
</div>
</el-upload>
<div class="progress-box" :style="{ opacity: upload.opacity }">
<!-- <span>上传中：</span> -->
<el-progress
:percentage="upload.rate"
:status="upload.rate === 100 ? 'success' : void 0"
/>
</div>
</el-form-item>
</el-form>
<!-- <div slot="footer" class="dialog-footer text-right">
<el-button @click="last">上一步</el-button>
<el-button type="primary" @click="next" :loading="loading"
>确定</el-button
>
</div> -->
</div>
</template>
<script>
// npm 下载 downloadjs
import downloadjs from 'downloadjs'
import { createMySql } from '@/api/dataSet'
import createAlluxio from '@/utils/createAlluxio'
import isUTF8 from '@/utils/validate/isUTF8'
import checkCsv from '@/utils/checkCsv'
export default {
props: {
sampleType: { type: String, required: true },
setLoading: { type: Function, required: true },
},
data() {
return {
form: { path: '', type: '1', tablename: '' },
path: null,
filename: '',
upload: {
opacity: 0,
rate: 0,
},
loading: false,
cancelUploadFile: null,
}
},
mounted() {
// createFilePath()
},
methods: {
cancel() {
this.$refs.form.resetFields()
this.$refs.upload.clearFiles()
this.cancelUploadFile && this.cancelUploadFile('取消上传')
this.upload.opacity = 0
this.upload.rate = 0
},
last() {
this.cancel()
this.$emit('last')
},
getForm() {
const { upload } = this
let formValid
this.$refs.form.validate(valid => {
formValid = valid
})
if (!formValid) {
return null
}
if (upload.rate !== 100) {
this.$message.warning('文件上传中，请稍作等待. . .')
return null
}
return { ...this.form }
},
next(baseForm) {
const { form, upload } = this
if (upload.opacity === 0 || !form.path) {
this.$message.warning('请选择上传文件')
return
}
if (upload.rate !== 100) {
this.$message.warning('文件上传中，请稍作等待. . .')
return
}
const params = { ...baseForm, ...form }
createMySql(params)
.then(() => {
this.$message.success('上传成功')
this.$emit('next')
})
.catch(() => {
this.$message.error('上传失败')
})
.finally(() => {
this.loading = false
this.path = null
this.filename = ''
this.upload.opacity = 0
this.upload.rate = 0
})
},
handleRemove() {
this.cancel()
},
handleExceed(files, fileList) {
this.$message.warning(
`当前限制选择 1 个文件，请先移移除已添加的${fileList[0].name}`,
)
},
async onFileChange(file, fileList) {
const { name } = file
const fileType = name.split('.').pop()
if (fileType !== 'csv') {
this.$message('上传文件类型只能为 csv 后缀的文件')
fileList.pop()
return
}
if (file.size === 0) {
this.$message('不能上传空文件')
fileList.pop()
return
}
const size = file.size / 1024 / 1024 / 1024
if (size > 1) {
this.$message('上传文件最大限制1G')
fileList.pop()
return
}
this.setLoading(true, '样本检测中...')
const fileContent = await new Promise(resolve => {
async function Uint8ArrayToString(u8arr) {
console.time('up')
const info = await checkCsv(u8arr)
console.timeEnd('up')
console.log(info)
return info
}
const reader = new FileReader()
reader.onload = async function(e) {
const data = e.target.result
const u8arr = new Uint8Array(data)
const isNoUTF8 = !isUTF8(u8arr)
const result = { isNoUTF8 }
if (!isNoUTF8) {
result.csvInfo = await Uint8ArrayToString(u8arr)
}
resolve(result)
}
reader.readAsArrayBuffer(file.raw)
})
if (fileContent.isNoUTF8) {
this.setLoading(false)
this.$message('上传文件编码只能为UTF8格式')
fileList.pop()
this.upload.opacity = 0
return
}
const csvInfo = fileContent.csvInfo
let isCsvError = false
if (csvInfo.rowError.length !== 0) {
this.setLoading(false)
isCsvError = true
this.$confirm('csv文件第一行为空', '提示', {
confirmButtonText: '确定',
showCancelButton: false,
type: 'warning',
})
.then(() => fileList.pop())
.catch(() => fileList.pop())
} else if (csvInfo.columnError.length !== 0) {
this.setLoading(false)
isCsvError = true
let text = `csv文件第一行第${csvInfo.columnError.join('、')}列为空`
if (csvInfo.columnError.length > 200) {
const arr = csvInfo.columnError.slice(0, 200)
text = `csv文件第一行总计${
csvInfo.bodyColumnError.length
}列为空：第${arr.join('、')}......列`
}
this.$confirm(text, '提示', {
confirmButtonText: '确定',
showCancelButton: false,
type: 'warning',
})
.then(() => fileList.pop())
.catch(() => fileList.pop())
} else if (!csvInfo.bodyStart) {
this.setLoading(false)
isCsvError = true
this.$confirm('csv文件数据为空', '提示', {
confirmButtonText: '确定',
showCancelButton: false,
type: 'warning',
})
.then(() => fileList.pop())
.catch(() => fileList.pop())
} else if (csvInfo.bodyColumnError.length !== 0) {
// this.setLoading(false)
isCsvError = true
let text = `csv文件第${csvInfo.bodyColumnError.join('、')}行列数不正确`
if (csvInfo.bodyColumnError.length > 200) {
const arr = csvInfo.bodyColumnError.slice(0, 200)
text = `csv文件总计${
csvInfo.bodyColumnError.length
}行列数不正确：第${arr.join('、')}......行`
}
this.$confirm(text, '提示', {
confirmButtonText: '确定',
showCancelButton: false,
type: 'warning',
})
.then(() => fileList.pop())
.catch(() => fileList.pop())
}
this.setLoading(false)
if (isCsvError) {
return
}
this.upload.opacity = 1
const alluxio = createAlluxio(
rate => {
this.upload.rate = rate
},
cancel => {
this.cancelUploadFile = cancel
},
)
const [err, filePath] = await alluxio.upload(file.raw)
if (err === null) {
this.form.path = filePath
this.form.tablename = name
this.$refs.form.validate()
this.$message.success('文件上传成功')
} else {
fileList.pop()
err !== '' && this.$message.error(err)
}
},
download() {
const name = this.sampleType === '1' ? 'Host' : 'Guest'
const path = `/excel/${name}.csv`
downloadjs(path)
},
},
}
</script>
<style lang="scss" scoped>
.el-upload__text-limit {
font-size: 14px;
font-weight: 400;
color: rgba(0, 0, 0, 0.45);
line-height: 22px;
}
.upload {
line-height: normal;
/deep/ .el-upload-list__item:first-child {
margin-top: 0;
}
}
/deep/ .el-button {
font-size: 12px;
}
</style>
五，step1页面中AddForm组件中的MySqlTypeForm
<template>
<div>
<el-form :model="sqlForm" label-width="100px" :rules="rules" ref="sqlForm">
<el-form-item label="数据源名称" prop="dataLinkName">
<el-input
v-model="sqlForm.dataLinkName"
placeholder="请输入数据源名称"
/>
</el-form-item>
<el-form-item label="数据源地址" prop="link">
<el-input v-model="sqlForm.link" placeholder="请输入数据源地址" />
</el-form-item>
<el-form-item label="数据源端口" prop="dataPort">
<el-input v-model="sqlForm.dataPort" placeholder="请输入数据源端口" />
</el-form-item>
<el-form-item label="数据库名称" prop="dataName">
<el-input v-model="sqlForm.dataName" placeholder="请输入数据库名称" />
</el-form-item>
<el-form-item label="数据库表" prop="tablename">
<el-input v-model="sqlForm.tablename" placeholder="请输入数据库表名" />
</el-form-item>
<el-form-item label="数据库用户名" prop="username">
<el-input v-model="sqlForm.username" placeholder="请输入数据库用户名" />
</el-form-item>
<el-form-item label="数据库密码" prop="password">
<el-input v-model="sqlForm.password" placeholder="请输入数据库密码" />
</el-form-item>
<el-form-item label="ID列" prop="labelId">
<el-input
v-model="sqlForm.labelId"
placeholder="请输入数据集中的ID列"
/>
</el-form-item>
<el-form-item label="标签列" prop="labelColumn">
<el-input
v-model="sqlForm.labelColumn"
placeholder="请输入数据集中的标签列"
/>
</el-form-item>
<div class="text-right" style=" 108px">
<el-button
type="text"
:loading="connectLoading"
@click="connect"
style="font-size:12px"
>连通测试</el-button
>
</div>
</el-form>
<!-- <div slot="footer" class="dialog-footer text-right">
<el-button @click="last">上一步</el-button>
<el-button type="primary" @click="next">确定</el-button>
</div> -->
</div>
</template>
<script>
import formRule, { formNameRule } from '@/utils/validate/formRule'
const formSqlRule = [
formRule.noBlank,
formRule.max,
{
pattern: /^[^\u4e00-\u9fa5]*$/,
message: '不能输入中文字符',
trigger: ['change', 'blur'],
},
{
pattern: /^[\w-]*$/,
message: '不能包含 “中划线，下划线” 之外的特殊字符',
trigger: ['change', 'blur'],
},
]
import { connectMySql, createMySql } from '@/api/dataSet'
const createForm = () => ({
dataLinkName: '',
link: '',
dataPort: '',
dataName: '',
tablename: '',
username: '',
passsword: '',
labelId: '',
labelColumn: '',
})
export default {
props: ['form', 'visible'],
data() {
return {
sqlForm: createForm(),
rules: {
dataLinkName: [
{ required: true, message: '请输入数据源名称', trigger: 'blur' },
...formNameRule,
],
link: [
{ required: true, message: '请输入数据源地址', trigger: 'blur' },
formRule.noBlank,
formRule.createMax(200),
],
dataPort: [
{ required: true, message: '请输入数据源端口', trigger: 'change' },
{
pattern: /^[0-9]{1,10}$/,
message: '只能输入10位及10位以内的数字',
trigger: ['change', 'blur'],
},
],
dataName: [
{ required: true, message: '请输入数据库名称', trigger: 'change' },
...formSqlRule,
],
tablename: [
{ required: true, message: '请输入数据库表名', trigger: 'change' },
...formSqlRule,
],
username: [
{ required: true, message: '请输入数据库用户名', trigger: 'change' },
...formSqlRule,
],
password: [
{ required: true, message: '请输入数据库密码', trigger: 'change' },
formRule.noBlank,
formRule.max,
{
pattern: /^[^\u4e00-\u9fa5]*$/,
message: '不能输入中文字符',
trigger: ['change', 'blur'],
},
],
labelId: [
{ required: false, message: '请输入ID列', trigger: 'change' },
...formSqlRule,
],
labelColumn: [...formSqlRule],
},
typeOptions: [
{ label: 'CSV', value: '1' },
{ label: 'MySql', value: '3' },
],
scopeOptions: [
{ label: '训练数据集', value: '1' },
{ label: '预测数据集', value: '2' },
],
connectLoading: false,
addLoading: false,
}
},
methods: {
getForm() {
let formValid
this.$refs.sqlForm.validate(valid => {
formValid = valid
})
if (formValid) {
return { ...this.sqlForm }
}
return null
},
next(baseForm) {
this.$refs.sqlForm.validate(valid => {
if (valid) {
this.addLoading = true
const params = { ...baseForm, ...this.sqlForm }
// params.username = params.userName
// delete params.userName
createMySql(params)
.then(res => {
console.log(res)
this.sqlForm = createForm()
this.$refs.sqlForm.resetFields()
this.addLoading = false
this.$message.success('添加成功')
this.$emit('next')
})
.catch(() => {
this.sqlForm = createForm()
this.addLoading = false
this.$message.error('添加失败')
})
}
})
},
last() {
this.sqlForm = createForm()
this.$refs.sqlForm.resetFields()
this.connectLoading = false
this.$emit('last')
},
connect() {
console.log(this.$refs.sqlForm)
this.$refs.sqlForm.validate(valid => {
if (valid) {
this.connectLoading = true
const { sqlForm } = this
const {
link,
dataPort,
dataName,
tablename,
username,
password,
labelId,
labelColumn,
} = sqlForm
const params = {
address: link + ':' + dataPort,
idColumnName: labelId,
labelColumnName: labelColumn,
tableName: tablename,
dataName,
username,
password,
}
connectMySql(params, {
timeout: 60000 * 60,
})
.then(() => {
if (this.connectLoading) {
this.$message.success('连接成功')
} else {
this.$message.success('连接失败')
}
})
.finally(() => {
this.connectLoading = false
})
}
})
},
},
}
</script>
六，step1页面中AddForm组件中的CsvTypeForm页面中createAlluxio组件
import axios from 'axios'
import { getAlluxioConfig } from '@/api/dataSet'
const CancelToken = axios.CancelToken
/**
* 接收两个 callback, 并返回 alluiox 实例，上传文件时调用实例上的 upload 方法
* @param {(rate: number) => undefined} onUploadProgress - 文件上传进度
* @param {(cancel: Function) => undefined} setCancel - 设置取消上传的回调
*
* @return {Alluxio}
* @typedef {Object} Alluxio
* @property {string} basePath - alluxio 服务地址
* @property {string} filePath - alluxio 文件存储路径
* @property {() => Promise<undefined>} getAlluxioConfig - 获取 alluxio 服务地址及文件存储路径
* @property {(fileName: string) => Promise<boolean>} isFile - 判断文件是否已存在
* @property {(fileName: string) => Promise<{fileId: number, fileName: string}>} createFile - 创建文件并返回文件id及文件名
* @property {(fileId: number, file: File) => Promise<undefined>} writeFile - 写入文件流
* @property {(fileId: number) => Promise<undefined>} closeFile - 关闭文件流
* @property {(file: File) => Promise<[?string, (string|undefined)]>} upload - 上传文件
*
*/
export default function createAlluxio(onUploadProgress, setCancel) {
const request = axios.create({
cancelToken: new CancelToken(cancel => {
setCancel(() => cancel('取消'))
}),
})
const versions = '/api/v1'
return {
basePath: '',
filePath: '',
async getAlluxioConfig() {
const {
data: { BASE_PATH, FILE_PATH },
} = await getAlluxioConfig(null, {
cancelToken: null,
})
let basePath = BASE_PATH.replace(/\/$/, '') + versions
let filePath = FILE_PATH.replace(/\/$/, '')
if (basePath.substring(0, 4) !== 'http') {
basePath = `http://${basePath}`
}
if (filePath.substring(0, 1) !== '/') {
filePath = `/${filePath}`
}
this.basePath = basePath
this.filePath = filePath
},
async isFile(fileName) {
const url = `${this.basePath}/paths/${this.filePath}/${fileName}/exists`
const { data } = await request.post(url, {})
return data
},
async createFile() {
let fileName = generateFileName()
while (await this.isFile(fileName)) {
fileName = generateFileName()
}
const url = `${this.basePath}/paths/${this.filePath}/${fileName}/create-file`
const { data } = await request.post(url, {})
return { fileId: data, fileName }
},
async writeFile(fileId, file) {
const url = `${this.basePath}/streams/${fileId}/write`
const { data } = await request.post(url, file, {
headers: { 'Content-Type': 'application/octet-stream' },
onUploadProgress({ loaded, total }) {
onUploadProgress(Math.floor((loaded / total) * 99))
},
})
return data
},
async closeFile(fileId) {
const url = `${this.basePath}/streams/${fileId}/close`
const { data } = await request.post(
url,
{},
{
onUploadProgress({ loaded, total }) {
onUploadProgress(Math.floor((loaded / total) * 1 + 99))
},
},
)
return data
},
async upload(file) {
await this.getAlluxioConfig()
try {
const { fileId, fileName } = await this.createFile()
await this.writeFile(fileId, file)
await this.closeFile(fileId)
return [null, `${this.filePath}/${fileName}`]
} catch (err) {
console.log(err)
// 请求超时或者网络有问题
if (err.message === '取消') {
// 取消重复请求 返回空字符串
return ['']
} else if (err.message.includes('timeout')) {
return ['请求超时！请检查网络是否正常']
} else if (err.message.includes('Network Error')) {
return ['网络连接错误']
} else {
return ['请求失败，请检查网络是否已连接']
}
}
},
}
}
/**
* 生成guid作为文件名称
* @return {string}
*/
function generateFileName() {
let d = new Date().getTime()
const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(
c,
) {
var r = (d + Math.random() * 16) % 16 | 0
d = Math.floor(d / 16)
return (c == 'x' ? r : (r & 0x7) | 0x8).toString(16)
})
return uuid + '.csv'
}
七，step1页面中AddForm组件中的CsvTypeForm页面中isUTF8组件
export default function isUTF8(bytes) {
// UTF-8 BOM
if (bytes[0] == 0xef && bytes[1] == 0xbb && bytes[2] == 0xbf) {
return false
}
let i = 0
while (i < bytes.length) {
if (
// ASCII
bytes[i] == 0x09 ||
bytes[i] == 0x0a ||
bytes[i] == 0x0d ||
(0x20 <= bytes[i] && bytes[i] <= 0x7e)
) {
i += 1
continue
}
if (
// non-overlong 2-byte
0xc2 <= bytes[i] &&
bytes[i] <= 0xdf &&
0x80 <= bytes[i + 1] &&
bytes[i + 1] <= 0xbf
) {
i += 2
continue
}
if (
// excluding overlongs
(bytes[i] == 0xe0 &&
0xa0 <= bytes[i + 1] &&
bytes[i + 1] <= 0xbf &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf) || // straight 3-byte
(((0xe1 <= bytes[i] && bytes[i] <= 0xec) ||
bytes[i] == 0xee ||
bytes[i] == 0xef) &&
0x80 <= bytes[i + 1] &&
bytes[i + 1] <= 0xbf &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf) || // excluding surrogates
(bytes[i] == 0xed &&
0x80 <= bytes[i + 1] &&
bytes[i + 1] <= 0x9f &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf)
) {
i += 3
continue
}
if (
// planes 1-3
(bytes[i] == 0xf0 &&
0x90 <= bytes[i + 1] &&
bytes[i + 1] <= 0xbf &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf &&
0x80 <= bytes[i + 3] &&
bytes[i + 3] <= 0xbf) || // planes 4-15
(0xf1 <= bytes[i] &&
bytes[i] <= 0xf3 &&
0x80 <= bytes[i + 1] &&
bytes[i + 1] <= 0xbf &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf &&
0x80 <= bytes[i + 3] &&
bytes[i + 3] <= 0xbf) || // plane 16
(bytes[i] == 0xf4 &&
0x80 <= bytes[i + 1] &&
bytes[i + 1] <= 0x8f &&
0x80 <= bytes[i + 2] &&
bytes[i + 2] <= 0xbf &&
0x80 <= bytes[i + 3] &&
bytes[i + 3] <= 0xbf)
) {
i += 4
continue
}
return false
}
return true
}
八，step1页面中AddForm组件中的CsvTypeForm页面中checkCsv组件
const LF_CODE = 10
const CR_CODE = 13
const SPACE_CODE = 32
const QUOTE_CODE = 34
const COMMA_CODE = 44
function checkSpace(start, end, u8arr) {
if (start === undefined) {
return true
}
for (let i = start; i < end; i++) {
if (u8arr[i] !== SPACE_CODE) {
return false
}
}
return true
}
function checkEmptyRow(index, u8arr) {
if (
index === 0 ||
u8arr[index - 1] === LF_CODE ||
u8arr[index - 1] === CR_CODE
) {
return true
}
return false
}
export function getHeadInfo(u8arr) {
let openQuote = false
let start = undefined
let end = undefined
const info = {
columnNumber: 0,
rowError: [],
columnError: [],
bodyStart: undefined,
}
for (let i = 0; i < u8arr.length; i++) {
if (!openQuote && start === undefined && u8arr[i] === QUOTE_CODE) {
openQuote = true
start = i + 1
} else if (openQuote) {
if (u8arr[i] === QUOTE_CODE) {
if (u8arr[i + 1] === QUOTE_CODE) {
i++
} else {
openQuote = false
end = i
}
} else if (u8arr[i] === LF_CODE || u8arr[i] === CR_CODE) {
openQuote = false
i--
}
} else if (u8arr[i] === COMMA_CODE) {
if (end === undefined) {
end = i
}
info.columnNumber += 1
if (checkSpace(start, end, u8arr)) {
info.columnError.push(info.columnNumber)
}
start = undefined
end = undefined
} else if (u8arr[i] === LF_CODE || u8arr[i] === CR_CODE) {
if (checkEmptyRow(i, u8arr)) {
info.rowError.push(1)
return info
} else {
if (end === undefined) {
end = i
}
info.columnNumber += 1
if (checkSpace(start, end, u8arr)) {
info.columnError.push(info.columnNumber)
}
}
if (u8arr[i] === CR_CODE && u8arr[i + 1] === LF_CODE) {
info.bodyStart = i + 2
} else {
info.bodyStart = i + 1
}
return info
} else if (start === undefined) {
start = i
}
}
return info
}
const sleep = () => new Promise(res => setTimeout(res, 3000))
/**
* 检测csv文件是否合规
* 通过遍历 uint8Array 判断 ascii 码识别行、列
* ascii code 10=\n 13=\r 34=" 44=,
* csv分行符有 \n， \r， \r\n 三种，escape characte
* 分列符 英文半角逗号 ','
* 单元格内容中出现分隔符时，单元格内容会被半角双引号包裹。 例："1,",2 转换为 ['1,', '2']
* 单元格内容中包含双引号时，需要使用2个双引号，第一个双引号作为转义符。例："1"",",2 转换为 ['1",', '2']
* @param {*} csv
* @returns
*/
export default async function checkCsv(u8arr) {
let openQuote = false
let start = undefined
let rowNumber = 1
let columnNumber = 0
const columnError = []
const rowError = []
const headInfo = getHeadInfo(u8arr)
const isheadInfo =
headInfo.rowError.length !== 0 || headInfo.columnError.length !== 0
if (!headInfo.bodyStart || isheadInfo) {
return headInfo
}
const sleep = () => new Promise(res => setTimeout(res, 0))
const headColumnNumber = headInfo.columnNumber
for (let i = headInfo.bodyStart; i < u8arr.length; i++) {
if (i % 1e7 === 0) {
await sleep()
}
if (!openQuote && start === undefined && u8arr[i] === QUOTE_CODE) {
openQuote = true
start = i + 1
} else if (openQuote) {
if (u8arr[i] === QUOTE_CODE) {
if (u8arr[i + 1] === QUOTE_CODE) {
i++
} else {
openQuote = false
}
} else if (u8arr[i] === LF_CODE || u8arr[i] === CR_CODE) {
openQuote = false
i--
}
} else if (u8arr[i] === COMMA_CODE) {
columnNumber += 1
start = undefined
} else if (u8arr[i] === LF_CODE || u8arr[i] === CR_CODE) {
rowNumber += 1
if (checkEmptyRow(i, u8arr)) {
rowError.push(rowNumber)
} else {
columnNumber += 1
if (headColumnNumber !== columnNumber) {
columnError.push(rowNumber)
}
columnNumber = 0
}
if (u8arr[i] === CR_CODE && u8arr[i + 1] === LF_CODE) {
i++
}
} else if (start === undefined) {
start = i
}
}
if (
u8arr[u8arr.length - 1] === LF_CODE ||
u8arr[u8arr.length - 1] === CR_CODE
) {
rowNumber += 1
rowError.push(rowNumber)
} else {
columnNumber += 1
if (headColumnNumber !== columnNumber) {
columnError.push(rowNumber)
}
}
headInfo.bodyRowError = rowError
headInfo.bodyColumnError = columnError
return headInfo
}