Skip to content

D
dlink
Unverified Commit 56a8db9a authored by aiwenmo's avatar aiwenmo Committed by GitHub
Browse files
Options

[Feature-728][admin,web] Add meta store feature (#729) 

Co-authored-by: 's avatarwenmo <32723967+wenmo@users.noreply.github.com>
parent 92f97a6a
Hide whitespace changes
Inline Side-by-side
Showing with 659 additions and 93 deletions
dlink-admin/src/main/java/com/dlink/controller/StudioController.java
View file @ 56a8db9a
package com.dlink.controller;
import com.dlink.assertion.Asserts;
import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.dlink.common.result.Result;
import com.dlink.dto.SessionDTO;
import com.dlink.dto.StudioCADTO;
import com.dlink.dto.StudioDDLDTO;
import com.dlink.dto.StudioExecuteDTO;
import com.dlink.explainer.lineage.LineageResult;
import com.dlink.dto.StudioMetaStoreDTO;
import com.dlink.job.JobResult;
import com.dlink.result.IResult;
import com.dlink.service.StudioService;
......@@ -14,12 +26,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.List;
/**
* StudioController
*
......@@ -161,4 +167,20 @@ public class StudioController {
@RequestParam String savePointType, @RequestParam String name, @RequestParam Integer taskId) {
return Result.succeed(studioService.savepoint(taskId, clusterId, jobId, savePointType, name), "savepoint 成功");
}
/**
* 获取 Meta Store Catalog 和 Database
*/
@PostMapping("/getMSCatalogs")
public Result getMSCatalogs(@RequestBody StudioMetaStoreDTO studioMetaStoreDTO) {
return Result.succeed(studioService.getMSCatalogs(studioMetaStoreDTO), "获取成功");
}
/**
* 获取 Meta Store Table
*/
@PostMapping("/getMSTables")
public Result getMSTables(@RequestBody StudioMetaStoreDTO studioMetaStoreDTO) {
return Result.succeed(studioService.getMSTables(studioMetaStoreDTO), "获取成功");
}
}
dlink-admin/src/main/java/com/dlink/dto/StudioMetaStoreDTO.java 0 → 100644
View file @ 56a8db9a
package com.dlink.dto;
import lombok.Getter;
import java.util.HashMap;
import java.util.Map;
import com.dlink.assertion.Asserts;
import com.dlink.gateway.GatewayType;
import com.dlink.job.JobConfig;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
/**
* StudioMetaStoreDTO
*
* @author wenmo
* @since 2022/7/16 23:18
*/
@Getter
public class StudioMetaStoreDTO extends AbstractStatementDTO {
private String catalog;
private String database;
private String dialect;
private Integer databaseId;
public JobConfig getJobConfig() {
return new JobConfig(
GatewayType.LOCAL.getLongValue(), true, false, false, false,
null, null, null, null, null,
null, isFragment(), false, false, 0,
null, null, null, null, null);
}
}
dlink-admin/src/main/java/com/dlink/service/StudioService.java
View file @ 56a8db9a
package com.dlink.service;
import com.dlink.dto.*;
import java.util.List;
import com.dlink.dto.SessionDTO;
import com.dlink.dto.SqlDTO;
import com.dlink.dto.StudioCADTO;
import com.dlink.dto.StudioDDLDTO;
import com.dlink.dto.StudioExecuteDTO;
import com.dlink.dto.StudioMetaStoreDTO;
import com.dlink.explainer.lineage.LineageResult;
import com.dlink.job.JobResult;
import com.dlink.model.Catalog;
import com.dlink.model.Table;
import com.dlink.result.IResult;
import com.dlink.result.SelectResult;
import com.dlink.result.SqlExplainResult;
......@@ -10,8 +19,6 @@ import com.dlink.session.SessionInfo;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.List;
/**
* StudioService
*
......@@ -47,4 +54,8 @@ public interface StudioService {
boolean cancel(Integer clusterId, String jobId);
boolean savepoint(Integer taskId, Integer clusterId, String jobId, String savePointType, String name);
List<Catalog> getMSCatalogs(StudioMetaStoreDTO studioMetaStoreDTO);
List<Table> getMSTables(StudioMetaStoreDTO studioMetaStoreDTO);
}
dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java
View file @ 56a8db9a
......@@ -2,6 +2,7 @@ package com.dlink.service.impl;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
......@@ -19,6 +20,7 @@ import com.dlink.dto.SqlDTO;
import com.dlink.dto.StudioCADTO;
import com.dlink.dto.StudioDDLDTO;
import com.dlink.dto.StudioExecuteDTO;
import com.dlink.dto.StudioMetaStoreDTO;
import com.dlink.explainer.lineage.LineageBuilder;
import com.dlink.explainer.lineage.LineageResult;
import com.dlink.gateway.GatewayType;
......@@ -29,10 +31,14 @@ import com.dlink.job.JobManager;
import com.dlink.job.JobResult;
import com.dlink.metadata.driver.Driver;
import com.dlink.metadata.result.JdbcSelectResult;
import com.dlink.model.Catalog;
import com.dlink.model.Cluster;
import com.dlink.model.DataBase;
import com.dlink.model.Savepoints;
import com.dlink.model.Schema;
import com.dlink.model.Table;
import com.dlink.model.Task;
import com.dlink.result.DDLResult;
import com.dlink.result.IResult;
import com.dlink.result.SelectResult;
import com.dlink.result.SqlExplainResult;
......@@ -115,6 +121,15 @@ public class StudioServiceImpl implements StudioService {
return jobResult;
}
private IResult executeMSFlinkSql(StudioMetaStoreDTO studioMetaStoreDTO) {
addFlinkSQLEnv(studioMetaStoreDTO);
JobConfig config = studioMetaStoreDTO.getJobConfig();
JobManager jobManager = JobManager.build(config);
IResult jobResult = jobManager.executeDDL(studioMetaStoreDTO.getStatement());
RunTimeUtil.recovery(jobManager);
return jobResult;
}
public JobResult executeCommonSql(SqlDTO sqlDTO) {
JobResult result = new JobResult();
result.setStatement(sqlDTO.getStatement());
......@@ -348,6 +363,84 @@ public class StudioServiceImpl implements StudioService {
return false;
}
@Override
public List<Catalog> getMSCatalogs(StudioMetaStoreDTO studioMetaStoreDTO) {
List<Catalog> catalogs = new ArrayList<>();
if (Dialect.isSql(studioMetaStoreDTO.getDialect())) {
DataBase dataBase = dataBaseService.getById(studioMetaStoreDTO.getDatabaseId());
if (!Asserts.isNull(dataBase)) {
Catalog defaultCatalog = Catalog.build("default_catalog");
Driver driver = Driver.build(dataBase.getDriverConfig());
defaultCatalog.setSchemas(driver.listSchemas());
catalogs.add(defaultCatalog);
}
} else {
studioMetaStoreDTO.setStatement("SHOW CATALOGS");
IResult result = executeMSFlinkSql(studioMetaStoreDTO);
if (result instanceof DDLResult) {
DDLResult ddlResult = (DDLResult) result;
Iterator<String> iterator = ddlResult.getColumns().iterator();
if (iterator.hasNext()) {
String key = iterator.next();
List<Map<String, Object>> rowData = ddlResult.getRowData();
for (Map<String, Object> item : rowData) {
catalogs.add(Catalog.build(item.get(key).toString()));
}
}
for (Catalog catalog : catalogs) {
String statement = "USE CATALOG " + catalog.getName() + ";\r\nSHOW DATABASES";
studioMetaStoreDTO.setStatement(statement);
IResult tableResult = executeMSFlinkSql(studioMetaStoreDTO);
if (result instanceof DDLResult) {
DDLResult tableDDLResult = (DDLResult) tableResult;
Iterator<String> tableIterator = tableDDLResult.getColumns().iterator();
if (tableIterator.hasNext()) {
String key = tableIterator.next();
List<Map<String, Object>> rowData = tableDDLResult.getRowData();
List<Schema> schemas = new ArrayList<>();
for (Map<String, Object> item : rowData) {
schemas.add(Schema.build(item.get(key).toString()));
}
catalog.setSchemas(schemas);
}
}
}
}
}
return catalogs;
}
@Override
public List<Table> getMSTables(StudioMetaStoreDTO studioMetaStoreDTO) {
List<Table> tables = new ArrayList<>();
if (Dialect.isSql(studioMetaStoreDTO.getDialect())) {
DataBase dataBase = dataBaseService.getById(studioMetaStoreDTO.getDatabaseId());
if (Asserts.isNotNull(dataBase)) {
Driver driver = Driver.build(dataBase.getDriverConfig());
tables.addAll(driver.listTables(studioMetaStoreDTO.getDatabase()));
}
} else {
String statement = "USE CATALOG " + studioMetaStoreDTO.getCatalog() + ";\r\n" +
"USE " + studioMetaStoreDTO.getDatabase() + ";\r\nSHOW TABLES";
studioMetaStoreDTO.setStatement(statement);
IResult result = executeMSFlinkSql(studioMetaStoreDTO);
if (result instanceof DDLResult) {
DDLResult ddlResult = (DDLResult) result;
Iterator<String> iterator = ddlResult.getColumns().iterator();
if (iterator.hasNext()) {
String key = iterator.next();
List<Map<String, Object>> rowData = ddlResult.getRowData();
for (Map<String, Object> item : rowData) {
Table table = Table.build(item.get(key).toString(), studioMetaStoreDTO.getDatabase());
table.setCatalog(studioMetaStoreDTO.getCatalog());
tables.add(table);
}
}
}
}
return tables;
}
private void initUDF(JobConfig config, String statement) {
if (!GatewayType.LOCAL.equalsValue(config.getType())) {
return;
......
dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java
View file @ 56a8db9a
package com.dlink.service.impl;
import cn.hutool.core.bean.BeanUtil;
import org.apache.commons.lang3.StringUtils;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.dlink.alert.Alert;
......@@ -70,25 +91,8 @@ import com.dlink.service.TaskVersionService;
import com.dlink.utils.CustomStringJavaCompiler;
import com.dlink.utils.JSONUtil;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import cn.hutool.core.bean.BeanUtil;
/**
* 任务 服务实现类
......@@ -185,9 +189,9 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
return executeCommonSql(SqlDTO.build(task.getStatement(),
task.getDatabaseId(), null));
}
if (StringUtils.isBlank(savePointPath)){
if (StringUtils.isBlank(savePointPath)) {
task.setSavePointStrategy(SavePointStrategy.LATEST.getValue());
}else {
} else {
task.setSavePointStrategy(SavePointStrategy.CUSTOM.getValue());
task.setSavePointPath(savePointPath);
updateById(task);
......@@ -202,7 +206,6 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
}
private JobResult executeCommonSql(SqlDTO sqlDTO) {
JobResult result = new JobResult();
result.setStatement(sqlDTO.getStatement());
......@@ -359,7 +362,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
defaultFlinkSQLEnvTask.setAlias("DefaultCatalog");
defaultFlinkSQLEnvTask.setDialect(Dialect.FLINKSQLENV.getValue());
StringBuilder sb = new StringBuilder();
sb.append("create catalog myCatalog with(\n");
sb.append("create catalog my_catalog with(\n");
sb.append(" 'type' = 'dlink_mysql',\n");
sb.append(" 'username' = '");
sb.append(username);
......@@ -372,7 +375,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
sb.append("'\n");
sb.append(")");
sb.append(separator);
sb.append("use catalog myCatalog");
sb.append("use catalog my_catalog");
sb.append(separator);
defaultFlinkSQLEnvTask.setStatement(sb.toString());
defaultFlinkSQLEnvTask.setFragment(true);
......@@ -545,11 +548,11 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
if (Asserts.isNotNull(task.getJobInstanceId()) && task.getJobInstanceId() != 0) {
savepointJobInstance(task.getJobInstanceId(), SavePointType.CANCEL.getValue());
}
if (StringUtils.isNotBlank(savePointPath)){
if (StringUtils.isNotBlank(savePointPath)) {
task.setSavePointStrategy(SavePointStrategy.CUSTOM.getValue());
task.setSavePointPath(savePointPath);
}
final JobResult jobResult = submitTaskToOnline(task, id);
final JobResult jobResult = submitTaskToOnline(task, id);
if (Job.JobStatus.SUCCESS == jobResult.getStatus()) {
task.setStep(JobLifeCycle.ONLINE.getValue());
task.setJobInstanceId(jobResult.getJobInstanceId());
......
dlink-common/src/main/java/com/dlink/model/Catalog.java 0 → 100644
View file @ 56a8db9a
package com.dlink.model;
import lombok.Getter;
import lombok.Setter;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
/**
* Catalog
*
* @author wenmo
* @since 2022/7/17 21:37
*/
@Getter
@Setter
public class Catalog implements Serializable {
private static final long serialVersionUID = -7535759384541414568L;
private String name;
private List<Schema> schemas = new ArrayList<>();
public Catalog() {
}
public Catalog(String name) {
this.name = name;
}
public Catalog(String name, List<Schema> schemas) {
this.name = name;
this.schemas = schemas;
}
public static Catalog build(String name) {
return new Catalog(name);
}
}
dlink-core/src/main/java/com/dlink/job/JobManager.java
View file @ 56a8db9a
......@@ -426,6 +426,7 @@ public class JobManager {
public IResult executeDDL(String statement) {
String[] statements = SqlUtil.getStatements(statement, sqlSeparator);
try {
IResult result = null;
for (String item : statements) {
String newStatement = executor.pretreatStatement(item);
if (newStatement.trim().isEmpty()) {
......@@ -437,10 +438,10 @@ public class JobManager {
}
LocalDateTime startTime = LocalDateTime.now();
TableResult tableResult = executor.executeSql(newStatement);
IResult result = ResultBuilder.build(operationType, config.getMaxRowNum(), false, false, executor.getTimeZone()).getResult(tableResult);
result = ResultBuilder.build(operationType, config.getMaxRowNum(), false, false, executor.getTimeZone()).getResult(tableResult);
result.setStartTime(startTime);
return result;
}
return result;
} catch (Exception e) {
e.printStackTrace();
}
......
dlink-web/src/components/Studio/StudioEdit/data.d.ts
View file @ 56a8db9a
......@@ -43,6 +43,15 @@ export type StudioParam = {
checkPoint?: number,
savePointPath?: string,
}
export type StudioMetaStoreParam = {
statement?: string,
fragment?: boolean,
dialect?: string,
envId?: number,
databaseId?: number,
}
export type CAParam = {
statement: string,
statementSet: boolean,
......
dlink-web/src/components/Studio/StudioEvent/DDL.ts
View file @ 56a8db9a
import {executeDDL} from "@/pages/DataStudio/service";
import {executeDDL, getMSCatalogs} from "@/pages/DataStudio/service";
import FlinkSQL from "./FlinkSQL";
import {SessionType} from "@/pages/DataStudio/model";
import {MetaStoreCatalogType, SessionType, TaskType} from "@/pages/DataStudio/model";
import {Modal, message} from "antd";
import {addOrUpdateData, getData, handleRemove} from "@/components/Common/crud";
......@@ -274,3 +274,35 @@ export function cancelTask(id: number) {
export function recoveryTask(id: number) {
return getData('api/task/recoveryTask', {id});
}
/*--- 刷新 MetaStore Catalogs ---*/
export async function showMetaStoreCatalogs(task: TaskType, dispatch: any) {
if (!task?.dialect) {
return;
}
let param = {
envId: task.envId,
fragment: task.fragment,
dialect: task.dialect,
databaseId: task.databaseId,
};
const result = getMSCatalogs(param);
result.then(res => {
const catalogs: MetaStoreCatalogType[] = [];
if (res.datas) {
for (let i = 0; i < res.datas.length; i++) {
catalogs.push({
name: res.datas[i].name,
databases: res.datas[i].schemas,
});
}
}
dispatch && dispatch({
type: "Studio/saveMetaStore",
payload: {
activeKey: task.id,
metaStore: catalogs
},
});
})
}
dlink-web/src/components/Studio/StudioLeftTool/StudioMetaStore/index.tsx 0 → 100644
View file @ 56a8db9a
import {Button, Col, Empty, message, Modal, Row, Select, Tabs, Tooltip, Tree} from "antd";
import {MetaStoreTableType, StateType} from "@/pages/DataStudio/model";
import {connect} from "umi";
import React, {useState} from "react";
import {
CodepenOutlined,
DownOutlined,
OrderedListOutlined,
ReloadOutlined,
TableOutlined
} from '@ant-design/icons';
import {Scrollbars} from 'react-custom-scrollbars';
import Columns from "@/pages/DataBase/Columns";
import Tables from "@/pages/DataBase/Tables";
import {TreeDataNode} from "@/components/Studio/StudioTree/Function";
import Generation from "@/pages/DataBase/Generation";
import {getMSTables} from "@/pages/DataStudio/service";
import {Dispatch} from "@@/plugin-dva/connect";
const {DirectoryTree} = Tree;
const {Option, OptGroup} = Select;
const {TabPane} = Tabs;
const StudioMetaStore = (props: any) => {
const {current, toolHeight} = props;
const [catalog, setCatalog] = useState<string>();
const [database, setDatabase] = useState<string>();
const [treeData, setTreeData] = useState<[]>([]);
const [modalVisit, setModalVisit] = useState(false);
const [row, setRow] = useState<TreeDataNode>();
const onRefreshTreeData = (catalogAndDatabase: string) => {
if (!current?.task?.dialect || !catalogAndDatabase) {
return;
}
const names = catalogAndDatabase.split('.');
let catalogTmp = 'default_catalog';
let databaseTmp = 'default_database';
if (names.length > 1) {
catalogTmp = names[0];
databaseTmp = names[1];
} else if (names.length == 1) {
databaseTmp = names[0];
}
setCatalog(catalogTmp);
setDatabase(databaseTmp);
let param = {
envId: current.task.envId,
fragment: current.task.fragment,
dialect: current.task.dialect,
databaseId: current.task.databaseId,
catalog: catalogTmp,
database: databaseTmp,
};
const result = getMSTables(param);
result.then(res => {
const tables: MetaStoreTableType[] = [];
if (res.datas) {
for (let i = 0; i < res.datas.length; i++) {
tables.push({
name: res.datas[i].name,
columns: res.datas[i].columns,
});
}
}
const tablesData: [] = [];
for (let i = 0; i < tables.length; i++) {
tablesData.push({
name: tables[i].name,
title: tables[i].name,
key: tables[i].name,
icon: <TableOutlined/>,
isLeaf: true,
catalog: catalogTmp,
database: databaseTmp,
})
}
setTreeData(tablesData);
props.saveMetaStoreTable(current.key, catalogTmp, databaseTmp, tables);
message.success(`刷新 Catalog 成功`);
})
};
const refreshMetaStoreTables = () => {
onRefreshTreeData(catalog + '.' + database);
};
const onChangeMetaStoreCatalogs = (value: number) => {
onRefreshTreeData(value);
};
const getMetaStoreCatalogsOptions = () => {
const itemList = [];
if (current?.metaStore) {
for (const item of current?.metaStore) {
itemList.push(<OptGroup label={item.name}>
{item.databases.map(({name}) => (
<Option value={item.name + '.' + name} label={item.name + '.' + name}>{name}</Option>
))}
</OptGroup>)
}
}
return itemList;
};
const openColumnInfo = (e: React.MouseEvent, node: TreeDataNode) => {
if (node.isLeaf) {
setRow(node);
setModalVisit(true);
}
}
const cancelHandle = () => {
setRow(undefined);
setModalVisit(false);
}
return (
<>
<Row>
<Col span={24}>
<Tooltip title="刷新 Catalog">
<Button
type="text"
icon={<ReloadOutlined/>}
onClick={refreshMetaStoreTables}
/>
</Tooltip>
</Col>
</Row>
<Select
style={{width: '95%'}}
placeholder="选择 Catalog & Database"
optionLabelProp="label"
onChange={onChangeMetaStoreCatalogs}
>
{getMetaStoreCatalogsOptions()}
</Select>
<Scrollbars style={{height: (toolHeight - 32)}}>
{treeData.length > 0 ? (
<DirectoryTree
showIcon
switcherIcon={<DownOutlined/>}
treeData={treeData}
onRightClick={({event, node}: any) => {
openColumnInfo(event, node)
}}
/>) : (<Empty image={Empty.PRESENTED_IMAGE_SIMPLE}/>)}
</Scrollbars>
<Modal
title={row?.key}
visible={modalVisit}
width={1000}
onCancel={() => {
cancelHandle();
}}
footer={[
<Button key="back" onClick={() => {
cancelHandle();
}}>
关闭
</Button>,
]}
>
<Tabs defaultActiveKey="tableInfo" size="small">
<TabPane
tab={
<span>
<TableOutlined/>
表信息
</span>
}
key="tableInfo"
>
{row ? <Tables table={row}/> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE}/>}
</TabPane>
<TabPane
tab={
<span>
<CodepenOutlined/>
字段信息
</span>
}
key="columnInfo"
>
{row ? <Columns dbId={current.task.databaseId} schema={row.database} table={row.name}/> :
<Empty image={Empty.PRESENTED_IMAGE_SIMPLE}/>}
</TabPane>
<TabPane
tab={
<span>
<OrderedListOutlined/>
SQL 生成
</span>
}
key="sqlGeneration"
>
{row ? <Generation dbId={current.task.databaseId} schema={row.database} table={row.name}/> :
<Empty image={Empty.PRESENTED_IMAGE_SIMPLE}/>}
</TabPane>
</Tabs>
</Modal>
</>
);
};
const mapDispatchToProps = (dispatch: Dispatch) => ({
saveMetaStoreTable: (activeKey: number, catalog: string, database: string, tables: MetaStoreTableType[]) => dispatch({
type: "Studio/saveMetaStoreTable",
payload: {
activeKey,
catalog,
database,
tables,
},
}),
});
export default connect(({Studio}: { Studio: StateType }) => ({
current: Studio.current,
toolHeight: Studio.toolHeight,
}), mapDispatchToProps)(StudioMetaStore);
dlink-web/src/components/Studio/StudioLeftTool/index.tsx
View file @ 56a8db9a
import {Tabs, Empty} from "antd";
import {BarsOutlined,DatabaseOutlined,AppstoreOutlined,ClusterOutlined,MessageOutlined,FireOutlined,FunctionOutlined} from "@ant-design/icons";
import {
BarsOutlined,
DatabaseOutlined,
AppstoreOutlined,
ClusterOutlined,
MessageOutlined,
InsertRowAboveOutlined,
FunctionOutlined
} from "@ant-design/icons";
import {StateType} from "@/pages/DataStudio/model";
import {connect} from "umi";
import styles from "./index.less";
......@@ -9,37 +17,41 @@ import React from "react";
import StudioDataBase from "./StudioDataBase";
import StudioCluster from "./StudioCluster";
import StudioMetaData from "./StudioMetaData";
import StudioMetaStore from "./StudioMetaStore";
const { TabPane } = Tabs;
const {TabPane} = Tabs;
const StudioLeftTool = (props:any) => {
const StudioLeftTool = (props: any) => {
const {toolHeight} = props;
return (
<Tabs defaultActiveKey="1" size="small" tabPosition="left" style={{ height: toolHeight}}>
<Tabs defaultActiveKey="1" size="small" tabPosition="left" style={{height: toolHeight}}>
<TabPane tab={<span><BarsOutlined/> 目录</span>} key="StudioTree">
<StudioTree/>
</TabPane>
<TabPane tab={<span><MessageOutlined /> 会话</span>} key="Connectors">
<StudioConnector />
<TabPane tab={<span><InsertRowAboveOutlined/> 结构</span>} key="MetaStore">
<StudioMetaStore/>
</TabPane>
<TabPane tab={<span><ClusterOutlined /> 集群</span>} key="Cluster">
<StudioCluster />
<TabPane tab={<span><MessageOutlined/> 会话</span>} key="Connectors">
<StudioConnector/>
</TabPane>
<TabPane tab={<span><DatabaseOutlined /> 数据源</span>} key="DataSource">
<StudioDataBase />
<TabPane tab={<span><ClusterOutlined/> 集群</span>} key="Cluster">
<StudioCluster/>
</TabPane>
<TabPane tab={<span><AppstoreOutlined /> 元数据</span>} key="MetaData">
<StudioMetaData />
<TabPane tab={<span><DatabaseOutlined/> 数据源</span>} key="DataSource">
<StudioDataBase/>
</TabPane>
<TabPane tab={<span><FunctionOutlined /> 函数</span>} key="Function" >
<Empty image={Empty.PRESENTED_IMAGE_SIMPLE} />
<TabPane tab={<span><AppstoreOutlined/> 元数据</span>} key="MetaData">
<StudioMetaData/>
</TabPane>
<TabPane tab={<span><FunctionOutlined/> 函数</span>} key="Function">
<Empty image={Empty.PRESENTED_IMAGE_SIMPLE}/>
</TabPane>
</Tabs>
);
};
export default connect(({ Studio }: { Studio: StateType }) => ({
export default connect(({Studio}: { Studio: StateType }) => ({
sql: Studio.sql,
toolHeight: Studio.toolHeight,
}))(StudioLeftTool);
dlink-web/src/components/Studio/StudioMenu/index.tsx
View file @ 56a8db9a
......@@ -483,7 +483,7 @@ const StudioMenu = (props: any) => {
return (
<Row className={styles.container}>
<Col span={24}>
{/*<Col span={24}>
<div>
<Space>
<Dropdown overlay={menu}>
......@@ -508,14 +508,14 @@ const StudioMenu = (props: any) => {
</Dropdown>
</Space>
</div>
</Col>
</Col>*/}
<Divider className={styles["ant-divider-horizontal-0"]}/>
<Col span={24}>
<Row>
<Col span={16}>
<Breadcrumb className={styles["dw-path"]}>
<EnvironmentOutlined/>
<Divider type="vertical"/>
<EnvironmentOutlined style={{lineHeight: '32px'}}/>
<Divider type="vertical" style={{height: 'unset'}}/>
{getPathItem(currentPath)}
</Breadcrumb>
{currentSession.session &&
......@@ -659,7 +659,7 @@ const StudioMenu = (props: any) => {
<Tooltip title="查看 API">
<Button
type="text"
icon={<ApiTwoTone />}
icon={<ApiTwoTone/>}
onClick={showAPI}
/>
</Tooltip>
......
dlink-web/src/components/Studio/StudioTree/index.tsx
View file @ 56a8db9a
......@@ -13,7 +13,7 @@ import UpdateCatalogueForm from './components/UpdateCatalogueForm';
import SimpleTaskForm from "@/components/Studio/StudioTree/components/SimpleTaskForm";
import {Scrollbars} from "react-custom-scrollbars";
import {getIcon} from "@/components/Studio/icon";
import {showEnv} from "@/components/Studio/StudioEvent/DDL";
import {showEnv, showMetaStoreCatalogs} from "@/components/Studio/StudioEvent/DDL";
import UploadModal from "@/components/Studio/StudioTree/components/UploadModal";
type StudioTreeProps = {
......@@ -170,7 +170,7 @@ const StudioTree: React.FC<StudioTreeProps> = (props) => {
toCut(rightClickNode);
} else if (key == 'Paste') {
toPaste(rightClickNode);
}else if (key == 'Copy') {
} else if (key == 'Copy') {
toCopy(rightClickNode);
}
};
......@@ -229,6 +229,7 @@ const StudioTree: React.FC<StudioTreeProps> = (props) => {
chart: {},
},
monaco: React.createRef(),
metaStore: []
};
newTabs!.activeKey = node!.taskId;
newTabs!.panes!.push(newPane);
......@@ -236,6 +237,7 @@ const StudioTree: React.FC<StudioTreeProps> = (props) => {
type: "Studio/saveTabs",
payload: newTabs,
});
showMetaStoreCatalogs(result.datas, dispatch);
})
}
};
......@@ -308,10 +310,10 @@ const StudioTree: React.FC<StudioTreeProps> = (props) => {
const toCopy = async (node: TreeDataNode | undefined) => {
let catalogues = {
taskId:node?.taskId,
taskId: node?.taskId,
parentId: node?.id
};
const datas = await handleOption('/api/catalogue/copyTask',"复制作业",catalogues);
const datas = await handleOption('/api/catalogue/copyTask', "复制作业", catalogues);
if (datas) {
getTreeData();
......@@ -385,7 +387,7 @@ const StudioTree: React.FC<StudioTreeProps> = (props) => {
<Menu.Item key='CreateCatalogue'>{'创建目录'}</Menu.Item>
<Menu.Item key='CreateTask'>{'创建作业'}</Menu.Item>
<Menu.Item key='Rename'>{'重命名'}</Menu.Item>
<Menu.Item key='Copy'>{'复制'}</Menu.Item>
<Menu.Item key='Copy'>{'复制'}</Menu.Item>
<Menu.Item key='Cut'>{'剪切'}</Menu.Item>
{cutId && <Menu.Item key='Paste'>{'粘贴'}</Menu.Item>}
<Menu.Item key='Delete'>{'删除'}</Menu.Item>
......
dlink-web/src/components/Studio/index.tsx
View file @ 56a8db9a
......@@ -24,9 +24,9 @@ const Studio = (props: any) => {
const [form] = Form.useForm();
const VIEW = {
leftToolWidth: 300,
marginTop: 116,
marginTop: 84,
topHeight: 35.6,
bottomHeight: 153.6,
bottomHeight: 127,
rightMargin: 32,
leftMargin: 36,
midMargin: 46,
......@@ -152,7 +152,6 @@ const Studio = (props: any) => {
</Row>
</DraggleVerticalLayout>
</Card>
</div>
)
};
......
dlink-web/src/pages/DataStudio/model.ts
View file @ 56a8db9a
import type {Effect, Reducer} from "umi";
import {
handleAddOrUpdate
handleAddOrUpdate
} from "@/components/Common/crud";
import type {SqlMetaData} from "@/components/Studio/StudioEvent/data";
......@@ -111,6 +111,7 @@ export type TabsItemType = {
monaco?: any;
isModified: boolean;
sqlMetaData?: SqlMetaData;
metaStore?: MetaStoreCatalogType[]
}
export type TabsType = {
......@@ -135,6 +136,26 @@ export type SessionType = {
connectors: ConnectorType[];
}
export type MetaStoreCatalogType = {
name: string;
databases: MetaStoreDataBaseType[];
}
export type MetaStoreDataBaseType = {
name: string;
tables: MetaStoreTableType[];
}
export type MetaStoreTableType = {
name: string;
columns: MetaStoreColumnType[];
}
export type MetaStoreColumnType = {
name: string;
type: string;
}
export type StateType = {
isFullScreen: boolean;
toolHeight?: number;
......@@ -174,6 +195,8 @@ export type ModelType = {
saveCurrentPath: Reducer<StateType>;
// saveMonaco: Reducer<StateType>;
saveSqlMetaData: Reducer<StateType>;
saveMetaStore: Reducer<StateType>;
saveMetaStoreTable: Reducer<StateType>;
saveTabs: Reducer<StateType>;
closeTabs: Reducer<StateType>;
changeActiveKey: Reducer<StateType>;
......@@ -219,10 +242,10 @@ const Model: ModelType = {
panes: [],
},
session: [],
result:{},
result: {},
rightClickMenu: false,
refs:{
history:{},
refs: {
history: {},
}
},
......@@ -250,12 +273,12 @@ const Model: ModelType = {
...state,
toolHeight: payload,
};
},saveToolRightWidth(state, {payload}) {
}, saveToolRightWidth(state, {payload}) {
return {
...state,
toolRightWidth: payload,
};
},saveToolLeftWidth(state, {payload}) {
}, saveToolLeftWidth(state, {payload}) {
return {
...state,
toolLeftWidth: payload,
......@@ -274,7 +297,7 @@ const Model: ModelType = {
return {
...state,
current: {...newCurrent},
tabs:{...newTabs},
tabs: {...newTabs},
};
},
saveCurrentPath(state, {payload}) {
......@@ -292,7 +315,7 @@ const Model: ModelType = {
saveSqlMetaData(state, {payload}) {
let newCurrent = state.current;
const newTabs = state.tabs;
if(newCurrent.key == payload.activeKey){
if (newCurrent.key == payload.activeKey) {
newCurrent.sqlMetaData = {...payload.sqlMetaData};
newCurrent.isModified = payload.isModified;
}
......@@ -309,6 +332,50 @@ const Model: ModelType = {
tabs: {...newTabs},
};
},
saveMetaStore(state, {payload}) {
let newCurrent = state.current;
const newTabs = state.tabs;
if (newCurrent.key == payload.activeKey) {
newCurrent.metaStore = [...payload.metaStore];
}
for (let i = 0; i < newTabs.panes.length; i++) {
if (newTabs.panes[i].key == payload.activeKey) {
newTabs.panes[i].metaStore = [...payload.metaStore];
break;
}
}
return {
...state,
current: {...newCurrent},
tabs: {...newTabs},
};
},
saveMetaStoreTable(state, {payload}) {
let newCurrent = state.current;
const newTabs = state.tabs;
if (newCurrent.key == payload.activeKey) {
for (let i = 0; i < newCurrent.metaStore.length; i++) {
if (newCurrent.metaStore[i].name === payload.catalog) {
for (let j = 0; j < newCurrent.metaStore[i].databases.length; j++) {
if (newCurrent.metaStore[i].databases[j].name === payload.database) {
newCurrent.metaStore[i].databases[j].tables = [...payload.tables]
}
}
}
}
}
for (let i = 0; i < newTabs.panes.length; i++) {
if (newTabs.panes[i].key == payload.activeKey) {
newTabs.panes[i] = {...newCurrent};
break;
}
}
return {
...state,
current: {...newCurrent},
tabs: {...newTabs},
};
},
saveTabs(state, {payload}) {
let newCurrent = state.current;
for (let i = 0; i < payload.panes.length; i++) {
......@@ -316,7 +383,7 @@ const Model: ModelType = {
newCurrent = payload.panes[i];
}
}
if(payload.panes.length == 0){
if (payload.panes.length == 0) {
return {
...state,
current: undefined,
......@@ -328,7 +395,7 @@ const Model: ModelType = {
...state,
current: {
...newCurrent,
isModified:false,
isModified: false,
},
tabs: {...payload},
currentPath: newCurrent.path,
......@@ -343,12 +410,12 @@ const Model: ModelType = {
}
}
let newCurrent = undefined;
if(newTabs.panes.length > 0){
if (newTabs.panes.length > 0) {
newCurrent = newTabs.panes[newTabs.panes.length - 1];
}
if (newCurrent && (newTabs.activeKey == payload)) {
newTabs.activeKey = newCurrent.key;
}else{
} else {
newTabs.activeKey = undefined;
}
return {
......@@ -400,7 +467,7 @@ const Model: ModelType = {
if (newTabs.panes[i].key == payload.key) {
newTabs.panes[i].task = payload;
newTabs.panes[i].isModified = false;
if(newCurrent.key == payload.key){
if (newCurrent.key == payload.key) {
newCurrent = newTabs.panes[i];
}
}
......@@ -446,7 +513,7 @@ const Model: ModelType = {
for (let i = 0; i < newTabs.panes.length; i++) {
if (newTabs.panes[i].key == payload.key) {
newTabs.panes[i].console.result.result = payload.datas;
if(newCurrent.key == payload.key){
if (newCurrent.key == payload.key) {
newCurrent.console = newTabs.panes[i].console;
}
break;
......@@ -463,27 +530,27 @@ const Model: ModelType = {
...state,
cluster: [...payload],
};
},saveSessionCluster(state, {payload}) {
}, saveSessionCluster(state, {payload}) {
return {
...state,
sessionCluster: [...payload],
};
},saveClusterConfiguration(state, {payload}) {
}, saveClusterConfiguration(state, {payload}) {
return {
...state,
clusterConfiguration: [...payload],
};
},saveDataBase(state, {payload}) {
}, saveDataBase(state, {payload}) {
return {
...state,
database: [...payload],
};
},saveEnv(state, {payload}) {
}, saveEnv(state, {payload}) {
return {
...state,
env: [...payload],
};
},saveChart(state, {payload}) {
}, saveChart(state, {payload}) {
let newTabs = state?.tabs;
let newCurrent = state?.current;
for (let i = 0; i < newTabs.panes.length; i++) {
......@@ -505,7 +572,7 @@ const Model: ModelType = {
for (let i = 0; i < newTabs.panes.length; i++) {
if (newTabs.panes[i].task.id == payload.id) {
newTabs.panes[i].task.step = payload.step;
if(newCurrent.key == newTabs.panes[i].key){
if (newCurrent.key == newTabs.panes[i].key) {
newCurrent = newTabs.panes[i];
}
}
......@@ -522,7 +589,7 @@ const Model: ModelType = {
for (let i = 0; i < newTabs.panes.length; i++) {
if (newTabs.panes[i].task.id == payload.id) {
newTabs.panes[i].task.jobInstanceId = payload.jobInstanceId;
if(newCurrent.key == newTabs.panes[i].key){
if (newCurrent.key == newTabs.panes[i].key) {
newCurrent = newTabs.panes[i];
}
}
......@@ -546,7 +613,7 @@ const Model: ModelType = {
newCurrent.task.alias = payload.name;
}
}
if(newTabs.panes.length == 0){
if (newTabs.panes.length == 0) {
return {
...state,
current: undefined,
......
dlink-web/src/pages/DataStudio/service.ts
View file @ 56a8db9a
import request from 'umi-request';
import {CAParam, StudioParam} from "@/components/Studio/StudioEdit/data";
import {CAParam, StudioMetaStoreParam, StudioParam} from "@/components/Studio/StudioEdit/data";
export async function executeSql(params: StudioParam) {
return request<API.Result>('/api/studio/executeSql', {
......@@ -46,7 +46,7 @@ export async function getJobPlan(params: StudioParam) {
});
}
export async function getJobData(jobId:string) {
export async function getJobData(jobId: string) {
return request<API.Result>('/api/studio/getJobData', {
method: 'GET',
params: {
......@@ -72,3 +72,21 @@ export async function getLineage(params: CAParam) {
},
});
}
export async function getMSCatalogs(params: StudioMetaStoreParam) {
return request<API.Result>('/api/studio/getMSCatalogs', {
method: 'POST',
data: {
...params,
},
});
}
export async function getMSTables(params: StudioMetaStoreParam) {
return request<API.Result>('/api/studio/getMSTables', {
method: 'POST',
data: {
...params,
},
});
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment