[Feature-318][core] Add column lineage from db sql

[Feature-318][core] Add column lineage from db sql

dlink-admin/src/main/java/com/dlink/controller/StudioController.java

 package com.dlink.controller;
 
+import com.dlink.assertion.Asserts;
 import com.dlink.common.result.Result;
 import com.dlink.dto.SessionDTO;
 import com.dlink.dto.StudioCADTO;
 import com.dlink.dto.StudioDDLDTO;
 import com.dlink.dto.StudioExecuteDTO;
+import com.dlink.explainer.lineage.LineageResult;
 import com.dlink.job.JobResult;
 import com.dlink.result.IResult;
 import com.dlink.service.StudioService;

dlink-admin/src/main/java/com/dlink/dto/StudioCADTO.java

@@ -15,4 +15,5 @@ public class StudioCADTO extends AbstractStatementDTO {
     // It's useless for the time being
     private Boolean statementSet;
     private Integer type;
+    private String dialect;
 }

dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java

@@ -256,8 +256,16 @@ public class StudioServiceImpl implements StudioService {
 
     @Override
     public LineageResult getLineage(StudioCADTO studioCADTO) {
-        addFlinkSQLEnv(studioCADTO);
-        return LineageBuilder.getLineage(studioCADTO.getStatement(), studioCADTO.getStatementSet());
+        if (Asserts.isNotNullString(studioCADTO.getDialect()) && !studioCADTO.getDialect().equalsIgnoreCase("flinksql")) {
+            if(studioCADTO.getDialect().equalsIgnoreCase("doris")){
+                return com.dlink.explainer.sqlLineage.LineageBuilder.getSqlLineage(studioCADTO.getStatement(),"mysql");
+            } else {
+                return com.dlink.explainer.sqlLineage.LineageBuilder.getSqlLineage(studioCADTO.getStatement(),studioCADTO.getDialect().toLowerCase());
+            }
+        } else {
+            addFlinkSQLEnv(studioCADTO);
+            return LineageBuilder.getLineage(studioCADTO.getStatement(), studioCADTO.getStatementSet());
+        }
     }
 
     @Override

dlink-core/pom.xml

@@ -147,5 +147,10 @@
             <scope>${scope.runtime}</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid-spring-boot-starter</artifactId>
+        </dependency>
+
     </dependencies>
-</project>
+</project>
\ No newline at end of file

dlink-core/src/main/java/com/dlink/explainer/sqlLineage/LineageColumn.java

+package com.dlink.explainer.sqlLineage;
+
+import com.dlink.assertion.Asserts;
+import lombok.Data;
+
+@Data
+public class LineageColumn implements Comparable<LineageColumn> {
+
+    public String getTargetColumnName() {
+        return targetColumnName;
+    }
+
+    public void setTargetColumnName(String targetColumnName) {
+        this.targetColumnName = targetColumnName;
+    }
+
+    private String targetColumnName;
+
+    private String sourceDbName;
+
+    public String getSourceDbName() {
+        return sourceDbName;
+    }
+
+    public void setSourceDbName(String sourceDbName) {
+        this.sourceDbName = sourceDbName;
+    }
+
+    public String getSourceTableName() {
+        return sourceTableName;
+    }
+
+    public String getSourceColumnName() {
+        return sourceColumnName;
+    }
+
+    public void setSourceColumnName(String sourceColumnName) {
+        this.sourceColumnName = sourceColumnName;
+    }
+
+    private String sourceTableName;
+
+    private String sourceColumnName;
+
+    public String getExpression() {
+        return expression;
+    }
+
+    public void setExpression(String expression) {
+        this.expression = expression;
+    }
+
+    private String expression;
+
+    public Boolean getIsEnd() {
+        return isEnd;
+    }
+
+    public void setIsEnd(Boolean end) {
+        isEnd = end;
+    }
+
+    private Boolean isEnd = false;
+
+    public void setSourceTableName(String sourceTableName) {
+        sourceTableName = Asserts.isNotNullString(sourceTableName) ? sourceTableName.replace("`",
+                "") : sourceTableName;
+        if(sourceTableName.contains(" ")){
+            sourceTableName = sourceTableName.substring(0,sourceTableName.indexOf(" "));
+        }
+        if (sourceTableName.contains(".")) {
+            if(Asserts.isNullString(this.sourceDbName)){
+                this.sourceDbName = sourceTableName.substring(0, sourceTableName.indexOf("."));
+            }
+//            this.sourceDbName = sourceTableName.substring(0, sourceTableName.indexOf("."));
+            this.sourceTableName = sourceTableName.substring(sourceTableName.indexOf(".") + 1);
+        } else {
+            this.sourceTableName = sourceTableName;
+        }
+    }
+
+    public int compareTo(LineageColumn o) {
+        if(Asserts.isNotNullString(this.getSourceDbName())&& Asserts.isNotNullString(this.getSourceTableName())){
+            if(this.getSourceDbName().equals(o.getSourceDbName())&&this.getSourceTableName().equals(o.getSourceTableName())&&this.getTargetColumnName().equals(o.getTargetColumnName())){
+                return 0;
+            }
+        } else if(Asserts.isNotNullString(this.getSourceTableName())){
+            if(this.getSourceTableName().equals(o.getSourceTableName())&&this.getTargetColumnName().equals(o.getTargetColumnName())){
+                return 0;
+            }
+        } else {
+            if (this.getTargetColumnName().equals(o.getTargetColumnName())) {
+                return 0;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        LineageColumn myColumn = (LineageColumn) o;
+
+        if (!this.getTargetColumnName().equals(myColumn.getTargetColumnName())) {
+            return false;
+        }
+
+        if (Asserts.isNotNullString(sourceTableName) && !sourceTableName.equals(myColumn.sourceTableName)) {
+            return false;
+        }
+
+        if (Asserts.isNotNullString(sourceColumnName)) {
+            return sourceColumnName.equals(myColumn.sourceColumnName);
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = getTargetColumnName().hashCode();
+
+        if (Asserts.isNotNullString(sourceTableName)) {
+            result = 31 * result + sourceTableName.hashCode();
+        }
+
+        if (Asserts.isNotNullString(sourceColumnName)) {
+            result = 31 * result + sourceColumnName.hashCode();
+        }
+
+        if (Asserts.isNotNullString(sourceDbName)) {
+            result = 31 * result + sourceDbName.hashCode();
+        }
+
+        return result;
+    }
+}

dlink-core/src/main/java/com/dlink/explainer/sqlLineage/TreeNode.java

+package com.dlink.explainer.sqlLineage;
+
+import java.util.*;
+
+public class TreeNode<T> implements Iterable<TreeNode<T>> {
+
+    /**
+     * 树节点
+     */
+    public T data;
+
+    /**
+     * 父节点，根没有父节点
+     */
+    public TreeNode<T> parent;
+
+    /**
+     * 子节点，叶子节点没有子节点
+     */
+    public List<TreeNode<T>> children;
+
+    /**
+     * 保存了当前节点及其所有子节点，方便查询
+     */
+    private List<TreeNode<T>> elementsIndex;
+
+    /**
+     * 构造函数
+     *
+     * @param data
+     */
+    public TreeNode(T data) {
+        this.data = data;
+        this.children = new LinkedList<TreeNode<T>>();
+        this.elementsIndex = new LinkedList<TreeNode<T>>();
+        this.elementsIndex.add(this);
+    }
+
+    public T getData() {
+        return data;
+    }
+
+    public List<TreeNode<T>> getChildren() {
+        return children;
+    }
+
+    /**
+     * 判断是否为根：根没有父节点
+     *
+     * @return
+     */
+    public boolean isRoot() {
+        return parent == null;
+    }
+
+    /**
+     * 判断是否为叶子节点：子节点没有子节点
+     *
+     * @return
+     */
+    public boolean isLeaf() {
+        return children.size() == 0;
+    }
+
+    /**
+     * 添加一个子节点
+     *
+     * @param child
+     * @return
+     */
+    public TreeNode<T> addChild(T child) {
+        TreeNode<T> childNode = new TreeNode<T>(child);
+        childNode.parent = this;
+        this.children.add(childNode);
+        this.registerChildForSearch(childNode);
+        return childNode;
+    }
+
+    public TreeNode<T> addChild(TreeNode childNode) {
+        childNode.parent = this;
+        this.children.add(childNode);
+        this.registerChildForSearch(childNode);
+        return childNode;
+    }
+
+    /**
+     * 获取当前节点的层
+     *
+     * @return
+     */
+    public int getLevel() {
+        if (this.isRoot()) {
+            return 0;
+        } else {
+            return parent.getLevel() + 1;
+        }
+    }
+
+    /**
+     * 递归为当前节点以及当前节点的所有父节点增加新的节点
+     *
+     * @param node
+     */
+    private void registerChildForSearch(TreeNode<T> node) {
+        elementsIndex.add(node);
+
+        if (parent != null) {
+            parent.registerChildForSearch(node);
+        }
+    }
+
+    /**
+     * 从当前节点及其所有子节点中搜索某节点
+     *
+     * @param cmp
+     * @return
+     */
+    public TreeNode<T> findTreeNode(Comparable<T> cmp) {
+        for (TreeNode<T> element : this.elementsIndex) {
+            T elData = element.data;
+
+            if (cmp.compareTo(elData) == 0) {
+                return element;
+            }
+        }
+
+        return null;
+    }
+
+    public TreeNode<T> findChildNode(Comparable<T> cmp) {
+        for (TreeNode<T> element : this.getChildren()) {
+            T elData = element.data;
+
+            if (cmp.compareTo(elData) == 0) {
+                return element;
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * 获取当前节点的迭代器
+     *
+     * @return
+     */
+    public Iterator<TreeNode<T>> iterator() {
+        TreeNodeIterator<T> iterator = new TreeNodeIterator<T>(this);
+        return iterator;
+    }
+
+    @Override
+    public String toString() {
+        return data != null ? data.toString() : "[tree data null]";
+    }
+
+    /**
+     * 获取所有叶子节点的数据
+     *
+     * @return
+     */
+    public Set<TreeNode<T>> getAllLeafs() {
+        Set<TreeNode<T>> leafNodes = new HashSet<TreeNode<T>>();
+
+        if (this.children.isEmpty()) {
+            leafNodes.add(this);
+        } else {
+            for (TreeNode<T> child : this.children) {
+                leafNodes.addAll(child.getAllLeafs());
+            }
+        }
+
+        return leafNodes;
+    }
+
+    /**
+     * 获取所有叶子节点的数据
+     *
+     * @return
+     */
+    public Set<T> getAllLeafData() {
+        Set<T> leafNodes = new HashSet<T>();
+
+        if (this.children.isEmpty()) {
+            leafNodes.add(this.data);
+        } else {
+            for (TreeNode<T> child : this.children) {
+                leafNodes.addAll(child.getAllLeafData());
+            }
+        }
+
+        return leafNodes;
+    }
+}

dlink-core/src/main/java/com/dlink/explainer/sqlLineage/TreeNodeIterator.java

+package com.dlink.explainer.sqlLineage;
+import java.util.Iterator;
+public class TreeNodeIterator<T> implements Iterator<TreeNode<T>> {
+
+    private ProcessStages doNext;
+    private TreeNode<T> next;
+    private Iterator<TreeNode<T>> childrenCurNodeIter;
+    private Iterator<TreeNode<T>> childrenSubNodeIter;
+    private TreeNode<T> treeNode;
+
+    public TreeNodeIterator(TreeNode<T> treeNode) {
+        this.treeNode = treeNode;
+        this.doNext = ProcessStages.ProcessParent;
+        this.childrenCurNodeIter = treeNode.children.iterator();
+    }
+
+    public boolean hasNext() {
+        if (this.doNext == ProcessStages.ProcessParent) {
+            this.next = this.treeNode;
+            this.doNext = ProcessStages.ProcessChildCurNode;
+            return true;
+        }
+
+        if (this.doNext == ProcessStages.ProcessChildCurNode) {
+            if (childrenCurNodeIter.hasNext()) {
+                TreeNode<T> childDirect = childrenCurNodeIter.next();
+                childrenSubNodeIter = childDirect.iterator();
+                this.doNext = ProcessStages.ProcessChildSubNode;
+                return hasNext();
+            } else {
+                this.doNext = null;
+                return false;
+            }
+        }
+
+        if (this.doNext == ProcessStages.ProcessChildSubNode) {
+            if (childrenSubNodeIter.hasNext()) {
+                this.next = childrenSubNodeIter.next();
+                return true;
+            } else {
+                this.next = null;
+                this.doNext = ProcessStages.ProcessChildCurNode;
+                return hasNext();
+            }
+        }
+
+        return false;
+    }
+
+    public TreeNode<T> next() {
+        return this.next;
+    }
+
+    /**
+     * 目前不支持删除节点
+     */
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+    enum ProcessStages {
+        ProcessParent, ProcessChildCurNode, ProcessChildSubNode
+    }
+}
+

dlink-web/src/components/Lineage/index.tsx

@@ -25,7 +25,6 @@ const Lineage = (props: any) => {
       tables: [],
       relations: []
     };
-    debugger;
     allData.relations.forEach(relation => {
       if (relation.srcTableId !== tableId) {
         return;

dlink-web/src/components/Studio/StudioConsole/StudioCA/index.tsx

@@ -18,6 +18,7 @@ const StudioCA = (props: any) => {
     const res = getLineage({
       statement:current.value,
       statementSet:current.task.statementSet,
+      dialect:current.task.dialect,
       type: 1,
     });
     res.then((result)=>{

dlink-web/src/components/Studio/StudioEdit/data.d.ts

@@ -47,4 +47,5 @@ export type CAParam = {
   statement: string,
   statementSet: boolean,
   type: number,
+  dialect?: string,
 }