From 91f43d66722ccddd0251746451c6d67d1736218b Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 31 Aug 2020 17:29:38 -0500 Subject: [PATCH 1/3] added skeleton of package for full python support --- data-extraction/pyDataExtraction/__init__.py | 1 + data-extraction/pyDataExtraction/__main__.py | 10 ++ .../pyDataExtraction/commonTypes/Graph.py | 76 ++++++++++ .../pyDataExtraction/commonTypes/Grid.py | 25 ++++ .../pyDataExtraction/commonTypes/Plotly.py | 15 ++ .../pyDataExtraction/commonTypes/Text.py | 64 +++++++++ .../pyDataExtraction/commonTypes/__init__.py | 8 ++ .../pyDataExtraction/commonTypes/base.py | 24 ++++ .../EvaluationEngine/PyEvalutationEngine.ts | 135 ++++++++++++++++++ 9 files changed, 358 insertions(+) create mode 100644 data-extraction/pyDataExtraction/__init__.py create mode 100644 data-extraction/pyDataExtraction/__main__.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/Graph.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/Grid.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/Plotly.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/Text.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/__init__.py create mode 100644 data-extraction/pyDataExtraction/commonTypes/base.py create mode 100644 extension/src/EvaluationWatchService/EvaluationEngine/PyEvalutationEngine.ts diff --git a/data-extraction/pyDataExtraction/__init__.py b/data-extraction/pyDataExtraction/__init__.py new file mode 100644 index 0000000..0df0a3c --- /dev/null +++ b/data-extraction/pyDataExtraction/__init__.py @@ -0,0 +1 @@ +import types diff --git a/data-extraction/pyDataExtraction/__main__.py b/data-extraction/pyDataExtraction/__main__.py new file mode 100644 index 0000000..6e00d0f --- /dev/null +++ b/data-extraction/pyDataExtraction/__main__.py @@ -0,0 +1,10 @@ +from json import dumps +from typing import Union, Dict, Optional +from abc import ABC, abstractmethod +from pyDataExtraction.commonTypes.Graph import Graph + +if __name__ == "__main__": + graph_data1 = {"A": ["B", "C"], "B": ["A,C"], "C": ["A,D"], "D": ["A"]} + graph_data2 = {1: [2, 3], 2: [1, 3], 3: [1, 4], 4: [1]} + graph = Graph(graph_data1) + print(graph) diff --git a/data-extraction/pyDataExtraction/commonTypes/Graph.py b/data-extraction/pyDataExtraction/commonTypes/Graph.py new file mode 100644 index 0000000..c9894c5 --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/Graph.py @@ -0,0 +1,76 @@ +from typing import Union, Dict, Optional +from pyDataExtraction.commonTypes.base import DataType + +# from docs +""" +interface NodeGraphData { + id: string; + label?: string; + color?: string; + shape?: "ellipse" | "box"; +} +""" +""" +interface EdgeGraphData { + from: string; + to: string; + label?: string; + id?: string; + color?: string; + dashes?: boolean; +} +""" +# NOTE: may not be able to encapsulate edge in separate class due to from being a syntax token +""" +class Edge: + + def __init__(self,from: str, to: str,): + self.fromnode + + def __repr__(self): + pass +""" +# NOTE: ran into issue Node object is not json serializable when ecapsulating in own class +""" +class Node(DataType): + def __init__(self, id: Union[int, str], label: Optional[str] = None): + super().__init__() + self.id = id + if label is None: + self.label = id + else: + self.label = label +""" + + +class Graph(DataType): + """ + + + Args: + DataType (Union[Dict[str,list],Dict[int,list]]): + either expects a dictionary with a list as values or a 2d array + some representation of a basic graph + """ + + def __init__(self, graph_data: Union[Dict[str, list], Dict[int, list]]): + super().__init__() + self.kind["graph"] = True + # TODO get working for both a dictionary and an nxn array + # self.nodes = [Node(node) for node in graph data] + self.nodes = [] + self.edges = [] + if isinstance(graph_data, dict): + # self.nodes = [Node(node) for node in graph data] + for node in graph_data: + self.nodes.append({"id": str(node)}) + # TODO change prints to log statements + # print("node: ", node) + # print("edges: ", graph_data[node]) + for edge in graph_data[node]: + # print(edge_i) + # print("edge: ", graph_data[node][edge_i]) + self.edges.append({"from": node, "to": edge}) + # edge_i += 1 + # edge_i = 0 + # node_i += 1 diff --git a/data-extraction/pyDataExtraction/commonTypes/Grid.py b/data-extraction/pyDataExtraction/commonTypes/Grid.py new file mode 100644 index 0000000..826101e --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/Grid.py @@ -0,0 +1,25 @@ +""" +export interface Grid { + kind: { array: true }; + columnLabels?: { label?: string }[]; + rows: { + label?: string; + columns: { + content?: string; + tag?: string; + color?: string; + }[]; + }[]; + markers?: { + id: string; + + row: number; + column: number; + rows?: number; + columns?: number; + + label?: string; + color?: string; + }[]; +} +""" diff --git a/data-extraction/pyDataExtraction/commonTypes/Plotly.py b/data-extraction/pyDataExtraction/commonTypes/Plotly.py new file mode 100644 index 0000000..036a638 --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/Plotly.py @@ -0,0 +1,15 @@ +from pyDataExtraction.commonTypes.base import DataType + +""" +export interface Plotly { + kind: { plotly: true }; + data: Partial[]; +} +// See plotly docs for Plotly.Data. +""" + + +class Plotly(DataType): + def __init__(self, data): + super().__init__() + self.kind["plotly"] = True diff --git a/data-extraction/pyDataExtraction/commonTypes/Text.py b/data-extraction/pyDataExtraction/commonTypes/Text.py new file mode 100644 index 0000000..33c8b58 --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/Text.py @@ -0,0 +1,64 @@ +from typing import Optional +from pyDataExtraction.commonTypes.base import DataType + +""" +interface Text { + kind: { text: true }; + text: string; + mimeType?: string; + fileName?: string; +} +""" + + +class Text(DataType): + def __init__( + self, + text_data: str, + mimeType: Optional[str] = None, + fileName: Optional[str] = None, + ): + super().__init__() + + self.kind["text"] = True + self.text = text_data + if mimeType is None: + self.mimeType = mimeType + if fileName is None: + self.fileName = fileName + + +""" +interface Svg extends Text { + kind: { text: true; svg: true }; +} +""" + + +class Svg(Text): + def __init__( + self, + text_data: str, + mimeType: Optional[str] = None, + fileName: Optional[str] = None, + ): + self.kind["svg"] = True + super().__init__(text_data, mimeType, fileName) + + +""" +interface DotGraph extends Text { + kind: { text: true; dotGraph: true }; +} +""" + + +class DotGraph(Text): + def __init__( + self, + text_data: str, + mimeType: Optional[str] = None, + fileName: Optional[str] = None, + ): + self.kind["dotGraph"] = True + super().__init__(text_data, mimeType, fileName) diff --git a/data-extraction/pyDataExtraction/commonTypes/__init__.py b/data-extraction/pyDataExtraction/commonTypes/__init__.py new file mode 100644 index 0000000..40da1b9 --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/__init__.py @@ -0,0 +1,8 @@ +from typing import Union, Dict, Optional +from abc import ABC, abstractmethod + +from pyDataExtraction.commonTypes import base +from pyDataExtraction.commonTypes import Graph +from pyDataExtraction.commonTypes import Grid +from pyDataExtraction.commonTypes import Plotly +from pyDataExtraction.commonTypes import Text diff --git a/data-extraction/pyDataExtraction/commonTypes/base.py b/data-extraction/pyDataExtraction/commonTypes/base.py new file mode 100644 index 0000000..22fb6e7 --- /dev/null +++ b/data-extraction/pyDataExtraction/commonTypes/base.py @@ -0,0 +1,24 @@ +from json import dumps +from abc import ABC, abstractmethod + +# import logging +# TODO fix the import structure +# TODO figure out how to set up logging across a python library + + +class DataType(ABC): + """Abstract class for all supported dataTypesdataTypes + + Args: + object ([type]): [description] + """ + + def __init__(self): + self.kind = {} + super().__init__() + + def __repr__(self): + """returns json object format when printed or using str() + """ + return dumps(self.__dict__) + diff --git a/extension/src/EvaluationWatchService/EvaluationEngine/PyEvalutationEngine.ts b/extension/src/EvaluationWatchService/EvaluationEngine/PyEvalutationEngine.ts new file mode 100644 index 0000000..f89704a --- /dev/null +++ b/extension/src/EvaluationWatchService/EvaluationEngine/PyEvalutationEngine.ts @@ -0,0 +1,135 @@ +import { + getExpressionForDataExtractorApi, + DataResult, + ApiHasNotBeenInitializedCode, + getExpressionToInitializeDataExtractorApi, + DataExtractionResult, +} from "@hediet/debug-visualizer-data-extraction"; +import { EnhancedDebugSession } from "../../debugger/EnhancedDebugSession"; +import { + EvaluationEngine, + Evaluator, + EvaluationArgs, +} from "./EvaluationEngine"; +import { FormattedMessage } from "../../webviewContract"; +import { registerUpdateReconciler, hotClass } from "@hediet/node-reload"; + +registerUpdateReconciler(module); + +@hotClass(module) +export class PyEvaluationEngine implements EvaluationEngine { + createEvaluator(session: EnhancedDebugSession): Evaluator | undefined { + const supportedDebugAdapters = [ + "python", + + ]; + if (supportedDebugAdapters.indexOf(session.session.type) !== -1) { + return new PyEvaluator(session); + } + return undefined; + } +} + +class PyEvaluator implements Evaluator { + public readonly languageId = "python"; + + constructor(private readonly session: EnhancedDebugSession) { } + + private getContext(): "copy" | "repl" { + if (this.session.session.type.startsWith("pwa-")) { + return "copy"; + } + return "repl"; + } + + public async evaluate({ + expression, + preferredExtractorId, + frameId, + }: EvaluationArgs): Promise< + | { kind: "data"; result: DataExtractionResult } + | { kind: "error"; message: FormattedMessage } + > { + while (true) { + try { + const preferredExtractorExpr = preferredExtractorId + ? `"${preferredExtractorId}"` + : "undefined"; + + const body = `${getExpressionForDataExtractorApi()}.getData( + e => (${expression}), + expr => eval(expr), + ${preferredExtractorExpr} + )`; + + const wrappedExpr = ` + (() => { + try { + return ${body}; + } catch (e) { + return JSON.stringify({ + kind: "Error", + message: e.message, + stack: e.stack + }); + } + })() + `; + + const reply = await this.session.evaluate({ + expression: wrappedExpr, + frameId, + context: this.getContext(), + }); + const resultStr = reply.result; + const jsonData = + this.getContext() === "copy" + ? resultStr + : resultStr.substr(1, resultStr.length - 2); + const result = JSON.parse(jsonData) as DataResult; + + if (result.kind === "NoExtractors") { + throw new Error("No extractors"); + } else if (result.kind === "Error") { + throw new Error(result.message); + } else if (result.kind === "Data") { + return { + kind: "data", + result: result.extractionResult, + }; + } else { + throw new Error("Invalid Data"); + } + } catch (error) { + const msg = error.message as string | undefined; + if (msg && msg.includes(ApiHasNotBeenInitializedCode)) { + if (await this.initializeApi(frameId)) { + continue; + } + } + + return { + kind: "error", + message: error.message, + }; + } + } + } + + private async initializeApi(frameId: number | undefined): Promise { + try { + // prefer existing is true, so that manually registered (possibly newer) extractors are not overwritten. + const expression = `${getExpressionToInitializeDataExtractorApi()}.registerDefaultExtractors(true);`; + + await this.session.evaluate({ + expression, + frameId, + context: this.getContext(), + }); + + return true; + } catch (error) { + return false; + } + } +} From 691525c7a3d5e02e8903cdb9aa02f3756be3edcb Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 2 Sep 2020 10:04:24 -0500 Subject: [PATCH 2/3] changes related to comments on pull request --- data-extraction/pyDataExtraction/__init__.py | 2 +- data-extraction/pyDataExtraction/__main__.py | 7 ++++--- data-extraction/pyDataExtraction/commonTypes/Graph.py | 11 ++--------- data-extraction/pyDataExtraction/commonTypes/Grid.py | 2 ++ data-extraction/pyDataExtraction/commonTypes/Text.py | 2 +- data-extraction/pyDataExtraction/commonTypes/base.py | 6 +++--- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/data-extraction/pyDataExtraction/__init__.py b/data-extraction/pyDataExtraction/__init__.py index 0df0a3c..1c99f5e 100644 --- a/data-extraction/pyDataExtraction/__init__.py +++ b/data-extraction/pyDataExtraction/__init__.py @@ -1 +1 @@ -import types +import pyDataExtraction.commonTypes diff --git a/data-extraction/pyDataExtraction/__main__.py b/data-extraction/pyDataExtraction/__main__.py index 6e00d0f..b1023ef 100644 --- a/data-extraction/pyDataExtraction/__main__.py +++ b/data-extraction/pyDataExtraction/__main__.py @@ -1,10 +1,11 @@ -from json import dumps +import json from typing import Union, Dict, Optional -from abc import ABC, abstractmethod +from abc import ABC + from pyDataExtraction.commonTypes.Graph import Graph if __name__ == "__main__": - graph_data1 = {"A": ["B", "C"], "B": ["A,C"], "C": ["A,D"], "D": ["A"]} + graph_data1 = {"A": ["B", "C"], "B": ["A", "C"], "C": ["A", "D"], "D": ["A"]} graph_data2 = {1: [2, 3], 2: [1, 3], 3: [1, 4], 4: [1]} graph = Graph(graph_data1) print(graph) diff --git a/data-extraction/pyDataExtraction/commonTypes/Graph.py b/data-extraction/pyDataExtraction/commonTypes/Graph.py index c9894c5..d0ccaff 100644 --- a/data-extraction/pyDataExtraction/commonTypes/Graph.py +++ b/data-extraction/pyDataExtraction/commonTypes/Graph.py @@ -44,11 +44,10 @@ def __init__(self, id: Union[int, str], label: Optional[str] = None): class Graph(DataType): - """ - + """An implementation of the Graph data type for the visualizer Args: - DataType (Union[Dict[str,list],Dict[int,list]]): + DataType (Union[Dict[str, list], Dict[int, list]]): either expects a dictionary with a list as values or a 2d array some representation of a basic graph """ @@ -57,20 +56,14 @@ def __init__(self, graph_data: Union[Dict[str, list], Dict[int, list]]): super().__init__() self.kind["graph"] = True # TODO get working for both a dictionary and an nxn array - # self.nodes = [Node(node) for node in graph data] self.nodes = [] self.edges = [] if isinstance(graph_data, dict): - # self.nodes = [Node(node) for node in graph data] for node in graph_data: self.nodes.append({"id": str(node)}) # TODO change prints to log statements # print("node: ", node) # print("edges: ", graph_data[node]) for edge in graph_data[node]: - # print(edge_i) # print("edge: ", graph_data[node][edge_i]) self.edges.append({"from": node, "to": edge}) - # edge_i += 1 - # edge_i = 0 - # node_i += 1 diff --git a/data-extraction/pyDataExtraction/commonTypes/Grid.py b/data-extraction/pyDataExtraction/commonTypes/Grid.py index 826101e..56aecc7 100644 --- a/data-extraction/pyDataExtraction/commonTypes/Grid.py +++ b/data-extraction/pyDataExtraction/commonTypes/Grid.py @@ -1,3 +1,5 @@ +from pyDataExtraction.commonTypes.base import DataType + """ export interface Grid { kind: { array: true }; diff --git a/data-extraction/pyDataExtraction/commonTypes/Text.py b/data-extraction/pyDataExtraction/commonTypes/Text.py index 33c8b58..2b9be3e 100644 --- a/data-extraction/pyDataExtraction/commonTypes/Text.py +++ b/data-extraction/pyDataExtraction/commonTypes/Text.py @@ -22,7 +22,7 @@ def __init__( self.kind["text"] = True self.text = text_data - if mimeType is None: + if mimeType is not None: self.mimeType = mimeType if fileName is None: self.fileName = fileName diff --git a/data-extraction/pyDataExtraction/commonTypes/base.py b/data-extraction/pyDataExtraction/commonTypes/base.py index 22fb6e7..9f03029 100644 --- a/data-extraction/pyDataExtraction/commonTypes/base.py +++ b/data-extraction/pyDataExtraction/commonTypes/base.py @@ -1,5 +1,5 @@ -from json import dumps -from abc import ABC, abstractmethod +import json +from abc import ABC # import logging # TODO fix the import structure @@ -20,5 +20,5 @@ def __init__(self): def __repr__(self): """returns json object format when printed or using str() """ - return dumps(self.__dict__) + return json.dumps(self.__dict__) From 607d6814e66e404c003ea46b3354c41d92c6cd70 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 2 Sep 2020 10:30:32 -0500 Subject: [PATCH 3/3] added comment to __main__ for clarification --- data-extraction/pyDataExtraction/__main__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data-extraction/pyDataExtraction/__main__.py b/data-extraction/pyDataExtraction/__main__.py index b1023ef..6b93350 100644 --- a/data-extraction/pyDataExtraction/__main__.py +++ b/data-extraction/pyDataExtraction/__main__.py @@ -5,6 +5,11 @@ from pyDataExtraction.commonTypes.Graph import Graph if __name__ == "__main__": + # NOTE this will likely be what will be called by the PyEvaluation Engine, + # this will be where this library interfaces with the existing codebase + + # TODO: implement testing for each dataType, it may be a good idea to compare + # with output of typescript extractors graph_data1 = {"A": ["B", "C"], "B": ["A", "C"], "C": ["A", "D"], "D": ["A"]} graph_data2 = {1: [2, 3], 2: [1, 3], 3: [1, 4], 4: [1]} graph = Graph(graph_data1)