Skip to content

Commit 20adc63

Browse files
aeberhartaeberhart
committed
LLM friendly documentation: add queries, functions
1 parent 86253c4 commit 20adc63

File tree

4 files changed

+398
-1
lines changed

4 files changed

+398
-1
lines changed

‎dashjoin-core/src/test/java/org/dashjoin/util/LLMs.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ public static void main(String[] args) throws Exception {
3636
b.append("========================\n\n");
3737

3838
for (File f : new File[] {new File("../dashjoin-docs/llms/widget.json"),
39-
new File("../dashjoin-docs/llms/input.json")}) {
39+
new File("../dashjoin-docs/llms/input.json"),
40+
new File("../dashjoin-docs/llms/config.json")}) {
4041
Map<String, List<Example>> list =
4142
om.readValue(f, new TypeReference<Map<String, List<Example>>>() {});
4243

@@ -58,6 +59,7 @@ public static void main(String[] args) throws Exception {
5859

5960
writeInput();
6061
writeWidget();
62+
writeConfig();
6163
}
6264

6365
static void writeInput() throws Exception {
@@ -83,6 +85,27 @@ static void writeInput() throws Exception {
8385
Charset.defaultCharset());
8486
}
8587

88+
static void writeConfig() throws Exception {
89+
StringBuffer b = new StringBuffer();
90+
b.append("# Appendix: Databases, Queries, Functions, and Configurations\n");
91+
b.append("These are possible children of the create, edit, button, and variable widgets.");
92+
Map<String, List<Example>> list = om.readValue(new File("../dashjoin-docs/llms/config.json"),
93+
new TypeReference<Map<String, List<Example>>>() {});
94+
95+
for (Entry<String, List<Example>> entry : list.entrySet())
96+
for (Example e : entry.getValue()) {
97+
if (e.language == null)
98+
e.language = e.code instanceof String ? "jsonata" : "json";
99+
b.append("## " + e.title + "\n");
100+
b.append(e.description + "\n");
101+
b.append("```json\n");
102+
b.append(om.writerWithDefaultPrettyPrinter().writeValueAsString(e.code) + "\n```\n");
103+
}
104+
105+
FileUtils.write(new File("../dashjoin-docs/docs/appendix-configurations.md"), b,
106+
Charset.defaultCharset());
107+
}
108+
86109
@SuppressWarnings("unchecked")
87110
static void writeWidget() throws Exception {
88111
StringBuffer b = new StringBuffer();

‎dashjoin-docs/docs/appendix-configurations.md‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Appendix: Databases, Queries, Functions, and Configurations
2+
These are possible children of the create, edit, button, and variable widgets.## query catalog
3+
a query on database 'dj/northwind' called 'group', executable for the 'authenticated' role
4+
```json
5+
{
6+
"ID" : "group",
7+
"database" : "dj/northwind",
8+
"query" : "SELECT CUSTOMERS.COUNTRY, COUNT(*) AS \"Number of Customers\" FROM CUSTOMERS GROUP BY CUSTOMERS.COUNTRY",
9+
"roles" : [ "authenticated" ],
10+
"type" : "read"
11+
}
12+
```
13+
## query catalog
14+
a query on database 'dj/northwind' called 'list'. Parameters limit and offset can be passed to the query
15+
```json
16+
{
17+
"ID" : "list",
18+
"database" : "dj/northwind",
19+
"query" : "select * from CATEGORIES limit ${limit} offset ${offset}",
20+
"type" : "read",
21+
"arguments" : {
22+
"limit" : {
23+
"type" : "number",
24+
"sample" : "5"
25+
},
26+
"offset" : {
27+
"type" : "number",
28+
"sample" : "0"
29+
}
30+
}
31+
}
32+
```
33+
## database definition
34+
postgres database connection information with encrypted password
35+
```json
36+
{
37+
"name" : "postgres",
38+
"djClassName" : "org.dashjoin.service.SQLDatabase",
39+
"username" : "postgres",
40+
"url" : "jdbc:postgresql://localhost:5432/postgres",
41+
"ID" : "dj/postgres",
42+
"password" : "DJ1#\b/gbzX8DDZa1lVaiLat0HdX9cDST2KHJk"
43+
}
44+
```
45+
## Function catalog: Invoke
46+
Function that adds two numbers passed in the argument object. It can be called via $call('add') or via REST
47+
```json
48+
{
49+
"ID" : "add",
50+
"djClassName" : "org.dashjoin.function.Invoke",
51+
"expression" : "{'result': x+y}",
52+
"roles" : [ "authenticated" ],
53+
"type" : "read"
54+
}
55+
```
56+
## Function catalog: RestJson
57+
Function that calls a web service. The fields of the function argument are used to construct the URL via from string template
58+
```json
59+
{
60+
"djClassName" : "org.dashjoin.function.RestJson",
61+
"url" : "https://api.geoapify.com/v1/geocode/search?street=${street}&postcode=${postcode}&city=${city}&country=${country}&apiKey=...",
62+
"method" : "GET",
63+
"contentType" : "application/json",
64+
"ID" : "address"
65+
}
66+
```
67+
## Function catalog: Credentials
68+
Encrypted credentials for OpenAI to be used in $curl and $chat functions
69+
```json
70+
{
71+
"ID" : "openai",
72+
"djClassName" : "org.dashjoin.function.Credentials",
73+
"username" : "Authorization",
74+
"password" : "DJ1#\b7Zw3EGtmVKaDuwwOtwXfWDG1y+awbon7WNQp9NmJ6EgUXEpYUMC8O7zRUw2kSnDxyATO0R3ke3NxjaT9zCwYyDGS5VDgYt/L",
75+
"apiKey" : true
76+
}
77+
```
78+
## Function catalog: ETL
79+
Extract load transform function. Loads the result of 'expression' into the database sqlite. The data is mapped using 'mappings'
80+
```json
81+
{
82+
"djClassName" : "com.dashjoin.function.ETL",
83+
"database" : "sqlite",
84+
"ID" : "misp",
85+
"type" : "write",
86+
"oldData" : "Delete All",
87+
"createSchema" : true,
88+
"mappings" : {
89+
"MISP_Event" : {
90+
"sourceTable" : "table",
91+
"extractColumn" : null,
92+
"extractKey" : null,
93+
"pk" : "uuid",
94+
"rowMapping" : null,
95+
"rowFilter" : null
96+
}
97+
},
98+
"expressions" : {
99+
"expression" : "$openJson(\"https://www.circl.lu/doc/misp/feed-osint/0b988513-9535-42f0-9ebc-5d6aec2e1c79.json\").Event.Attribute"
100+
}
101+
}
102+
```
103+
## Function catalog: Email
104+
Configures an SMTP server
105+
```json
106+
{
107+
"djClassName" : "org.dashjoin.function.Email",
108+
"ID" : "email",
109+
"type" : "write",
110+
"properties" : {
111+
"mail.smtp.port" : "25"
112+
},
113+
"username" : "user",
114+
"password" : "DJ1#\btW06MCaBJjnRvgvGgTaTpQ=="
115+
}
116+
```

‎dashjoin-docs/llms/config.json‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
{
2+
"dj-query-catalog": [
3+
{
4+
"title": "query catalog",
5+
"description": "a query on database 'dj/northwind' called 'group', executable for the 'authenticated' role",
6+
"file": "model/dj-query-catalog/group.json",
7+
"code": {
8+
"ID": "group",
9+
"database": "dj/northwind",
10+
"query": "SELECT CUSTOMERS.COUNTRY, COUNT(*) AS \"Number of Customers\" FROM CUSTOMERS GROUP BY CUSTOMERS.COUNTRY",
11+
"roles": [
12+
"authenticated"
13+
],
14+
"type": "read"
15+
}
16+
},
17+
{
18+
"title": "query catalog",
19+
"description": "a query on database 'dj/northwind' called 'list'. Parameters limit and offset can be passed to the query",
20+
"file": "model/dj-query-catalog/list.json",
21+
"code": {
22+
"ID": "list",
23+
"database": "dj/northwind",
24+
"query": "select * from CATEGORIES limit ${limit} offset ${offset}",
25+
"type": "read",
26+
"arguments": {
27+
"limit": {
28+
"type": "number",
29+
"sample": "5"
30+
},
31+
"offset": {
32+
"type": "number",
33+
"sample": "0"
34+
}
35+
}
36+
}
37+
}
38+
],
39+
"dj-database": [
40+
{
41+
"title": "database definition",
42+
"description": "postgres database connection information with encrypted password",
43+
"file": "model/dj-database/dj%2Fpostgres.json",
44+
"code": {
45+
"name": "postgres",
46+
"djClassName": "org.dashjoin.service.SQLDatabase",
47+
"username": "postgres",
48+
"url": "jdbc:postgresql://localhost:5432/postgres",
49+
"ID": "dj/postgres",
50+
"password": "DJ1#\b/gbzX8DDZa1lVaiLat0HdX9cDST2KHJk"
51+
}
52+
}
53+
],
54+
"dj-function": [
55+
{
56+
"title": "Function catalog: Invoke",
57+
"description": "Function that adds two numbers passed in the argument object. It can be called via $call('add') or via REST",
58+
"file": "model/dj-function/add.json",
59+
"code": {
60+
"ID": "add",
61+
"djClassName": "org.dashjoin.function.Invoke",
62+
"expression": "{'result': x+y}",
63+
"roles": [
64+
"authenticated"
65+
],
66+
"type": "read"
67+
}
68+
},
69+
{
70+
"title": "Function catalog: RestJson",
71+
"description": "Function that calls a web service. The fields of the function argument are used to construct the URL via from string template",
72+
"file": "model/dj-function/address.json",
73+
"code": {
74+
"djClassName": "org.dashjoin.function.RestJson",
75+
"url": "https://api.geoapify.com/v1/geocode/search?street=${street}&postcode=${postcode}&city=${city}&country=${country}&apiKey=...",
76+
"method": "GET",
77+
"contentType": "application/json",
78+
"ID": "address"
79+
}
80+
},
81+
{
82+
"title": "Function catalog: Credentials",
83+
"description": "Encrypted credentials for OpenAI to be used in $curl and $chat functions",
84+
"file": "model/dj-function/openai.json",
85+
"code": {
86+
"ID": "openai",
87+
"djClassName": "org.dashjoin.function.Credentials",
88+
"username": "Authorization",
89+
"password": "DJ1#\b7Zw3EGtmVKaDuwwOtwXfWDG1y+awbon7WNQp9NmJ6EgUXEpYUMC8O7zRUw2kSnDxyATO0R3ke3NxjaT9zCwYyDGS5VDgYt/L",
90+
"apiKey": true
91+
}
92+
},
93+
{
94+
"title": "Function catalog: ETL",
95+
"description": "Extract load transform function. Loads the result of 'expression' into the database sqlite. The data is mapped using 'mappings'",
96+
"file": "model/dj-function/misp.json",
97+
"code": {
98+
"djClassName": "com.dashjoin.function.ETL",
99+
"database": "sqlite",
100+
"ID": "misp",
101+
"type": "write",
102+
"oldData": "Delete All",
103+
"createSchema": true,
104+
"mappings": {
105+
"MISP_Event": {
106+
"sourceTable": "table",
107+
"extractColumn": null,
108+
"extractKey": null,
109+
"pk": "uuid",
110+
"rowMapping": null,
111+
"rowFilter": null
112+
}
113+
},
114+
"expressions": {
115+
"expression": "$openJson(\"https://www.circl.lu/doc/misp/feed-osint/0b988513-9535-42f0-9ebc-5d6aec2e1c79.json\").Event.Attribute"
116+
}
117+
}
118+
},
119+
{
120+
"title": "Function catalog: Email",
121+
"description": "Configures an SMTP server",
122+
"file": "model/dj-function/email.json",
123+
"code": {
124+
"djClassName": "org.dashjoin.function.Email",
125+
"ID": "email",
126+
"type": "write",
127+
"properties": {
128+
"mail.smtp.port": "25"
129+
},
130+
"username": "user",
131+
"password": "DJ1#\btW06MCaBJjnRvgvGgTaTpQ=="
132+
}
133+
}
134+
]
135+
}

0 commit comments

Comments
 (0)