Create a Catalog
This API creates a new catalog entity.
Syntax
POST /api/v3/catalog
Request Input
The CatalogEntity is one of the following:
Input Restrictions
- The supplied catalog entity must not contain any Dremio-generated fields such
id
ortag
. - For Datasets, only Virtual Datasets can be created. The supplied path determines where the dataset is stored.
- For Folders, folders can only be created in the user’s Home Space or any Spaces that the user has write access to.
Response Output
The CatalogEntity is one of the following:
Response Codes
400
- The supplied CatalogEntity object is invalid.403
- User does not have permission to create the catalog entity.409
- A catalog entity with the specified path already exists.
Example: Post Catalog for Sources
In this example, a new Elasticsearch source entity, my_elasticsearch, is created. The my_elasticsearch source has folder called yelp with four (4) physical datasets (PDS) (business, checkin, review, and user).
Note:
Postman was used to generate samples.
HTTP Request
POST localhost:9047/api/v3/catalog
Raw Body Input
{
"entityType": "source",
"config": {
"scriptsEnabled": true,
"showHiddenIndices": false,
"showIdColumn": false,
"readTimeoutMillis": 60000,
"scrollTimeoutMillis": 300000,
"usePainless": true,
"scrollSize": 4000,
"allowPushdownOnNormalizedOrAnalyzedFields": false,
"warnOnRowCountMismatch": false,
"encryptionValidationMode": "CERTIFICATE_AND_HOSTNAME_VALIDATION",
"hostList": [
{
"hostname": "172.25.0.94",
"port": 9200
}
],
"authenticationType": "ANONYMOUS",
"sslEnabled": false,
"useWhitelist": false
},
"type": "ELASTIC",
"name": "my_elasticsearch",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false
}
Curl
curl -X POST \
http://localhost:9047/api/v3/catalog \
-H 'Authorization: _dremioo8opojj6vn4ughkvcpalpr46d6' \
-H 'Content-Type: application/json' \
-d '{
"entityType": "source",
"config": {
"scriptsEnabled": true,
"showHiddenIndices": false,
"showIdColumn": false,
"readTimeoutMillis": 60000,
"scrollTimeoutMillis": 300000,
"usePainless": true,
"scrollSize": 4000,
"allowPushdownOnNormalizedOrAnalyzedFields": false,
"warnOnRowCountMismatch": false,
"encryptionValidationMode": "CERTIFICATE_AND_HOSTNAME_VALIDATION",
"hostList": [
{
"hostname": "172.25.0.94",
"port": 9200
}
],
"authenticationType": "ANONYMOUS",
"sslEnabled": false,
"useWhitelist": false
},
"type": "ELASTIC",
"name": "my_elasticsearch",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false
}'
Python
import requests
url = "http://localhost:9047/api/v3/catalog"
payload = "{\n \"entityType\": \"source\",\n \"config\": {\n \"scriptsEnabled\": true,\n \"showHiddenIndices\": false,\n \"showIdColumn\": false,\n \"readTimeoutMillis\": 60000,\n \"scrollTimeoutMillis\": 300000,\n \"usePainless\": true,\n \"scrollSize\": 4000,\n \"allowPushdownOnNormalizedOrAnalyzedFields\": false,\n \"warnOnRowCountMismatch\": false,\n \"encryptionValidationMode\": \"CERTIFICATE_AND_HOSTNAME_VALIDATION\",\n \"hostList\": [\n {\n \"hostname\": \"172.25.0.94\",\n \"port\": 9200\n }\n ],\n \"authenticationType\": \"ANONYMOUS\",\n \"sslEnabled\": false,\n \"useWhitelist\": false\n },\n \"type\": \"ELASTIC\",\n \"name\": \"my_elasticsearch\",\n \"metadataPolicy\": {\n \"authTTLMs\": 86400000,\n \"namesRefreshMs\": 3600000,\n \"datasetRefreshAfterMs\": 3600000,\n \"datasetExpireAfterMs\": 10800000,\n \"datasetUpdateMode\": \"PREFETCH_QUERIED\",\n \"deleteUnavailableDatasets\": true,\n \"autoPromoteDatasets\": false\n },\n \"accelerationGracePeriodMs\": 10800000,\n \"accelerationRefreshPeriodMs\": 3600000,\n \"accelerationNeverExpire\": false,\n \"accelerationNeverRefresh\": false\n}"
headers = {
'Authorization': "_dremioo8opojj6vn4ughkvcpalpr46d6",
'Content-Type': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
Response
{
"entityType": "source",
"config": {
"scriptsEnabled": true,
"showHiddenIndices": false,
"showIdColumn": false,
"readTimeoutMillis": 60000,
"scrollTimeoutMillis": 300000,
"usePainless": true,
"scrollSize": 4000,
"allowPushdownOnNormalizedOrAnalyzedFields": false,
"warnOnRowCountMismatch": false,
"encryptionValidationMode": "CERTIFICATE_AND_HOSTNAME_VALIDATION",
"hostList": [
{
"hostname": "172.25.0.94",
"port": 9200
}
],
"password": "$DREMIO_EXISTING_VALUE$",
"authenticationType": "ANONYMOUS",
"sslEnabled": false,
"useWhitelist": false
},
"state": {
"status": "warn",
"messages": [
{
"level": "WARN",
"message": "Elastic cluster health is yellow."
}
]
},
"id": "a7fe6900-a9d1-4afe-aeb7-3dc5ddfca5b7",
"tag": "0",
"type": "ELASTIC",
"name": "my_elasticsearch",
"createdAt": "2019-03-26T00:16:50.145Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "0477f384-28ee-49e3-9dc9-9dbaf77f40d1",
"path": [
"my_elasticsearch",
"yelp"
],
"tag": "0",
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"accessControlList": {
"version": "0"
}
}
Example: Post Catalog for Spaces
In this example, a new space entity, Pebbles, is created.
Note:
Postman was used to generate samples.
HTTP Request
POST localhost:9047/api/v3/catalog
Raw Body Input
{
"entityType": "space",
"name": "Pebbles"
}
Curl
curl -X POST \
http://localhost:9047/api/v3/catalog \
-H 'Authorization: _dremioo8opojj6vn4ughkvcpalpr46d6' \
-H 'Content-Type: application/json' \
-d '{
"entityType": "space",
"name": "Pebbles"
}'
Python
import requests
url = "http://localhost:9047/api/v3/catalog"
payload = "{\n \"entityType\": \"space\",\n \"name\": \"Pebbles\"\n}"
headers = {
'Authorization': "_dremioo8opojj6vn4ughkvcpalpr46d6",
'Content-Type': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
Response
{
"entityType": "space",
"id": "35f9ffe7-fd02-4509-aa99-ce626f123537",
"name": "Pebbles",
"tag": "0",
"accessControlList": {
"version": "0"
}
}
Example: Post catalog for Folders
In this example, a new folder entity, Color, is created in the Pebbles space.
Note:
Postman was used to generate samples.
HTTP Request
POST localhost:9047/api/v3/catalog
Raw Body Input
{
"entityType": "folder",
"path": [
"Pebbles",
"Color"
]
}
Curl
curl -X POST \
http://localhost:9047/api/v3/catalog \
-H 'Authorization: _dremioo8opojj6vn4ughkvcpalpr46d6' \
-H 'Content-Type: application/json' \
-d '{
"entityType": "folder",
"path": [
"Pebbles",
"Color"
]
}'
Python
import requests
url = "http://localhost:9047/api/v3/catalog"
payload = "{\n \"entityType\": \"folder\",\n \"path\": [\n \"Pebbles\",\n \"Color\"\n ]\n}"
headers = {
'Authorization': "_dremioo8opojj6vn4ughkvcpalpr46d6",
'Content-Type': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
Response
{
"entityType": "folder",
"id": "37732cdf-c491-4a15-afbf-f0e0a7563e40",
"path": [
"Pebbles",
"Color"
],
"tag": "0",
"accessControlList": {
"version": "0"
}
}
Example: Post Catalog for Datasets
In this example, a new virtual dataset entity (VDS) is created.
path
specifies the space, folder, and new dataset name:- Pebbles is the space.
- Color is the folder.
- my_elastic_dataset is the new VDS name.
type
specifies the type of dataset. In this case, a virtual dataset.sql
specifies the SQL query against the dataset that you are using to create information for the new dataset. In this case, all contents are selected against the review PDS.sqlContext
specifies the location of the dataset. In this case, my_elasticsearch is the source and yelp is the folder where the review dataset is located.
Note:
Postman was used to generate samples.
HTTP Request
POST localhost:9047/api/v3/catalog
Raw Body Input
{
"entityType": "dataset",
"path": [
"Pebbles",
"Color",
"my_elastic_dataset"
],
"type": "VIRTUAL_DATASET",
"sql": "select * from review",
"sqlContext": ["my_elasticsearch", "yelp"]
}
Curl
curl -X POST \
http://localhost:9047/api/v3/catalog \
-H 'Authorization: _dremioo8opojj6vn4ughkvcpalpr46d6' \
-H 'Content-Type: application/json' \
-d '{
"entityType": "dataset",
"path": [
"Pebbles",
"Color",
"my_elastic_dataset"
],
"type": "VIRTUAL_DATASET",
"sql": "select * from review",
"sqlContext": ["my_elasticsearch", "yelp"]
}'
Python
import requests
url = "http://localhost:9047/api/v3/catalog"
payload = "{\n \"entityType\": \"dataset\",\n \"path\": [\n \"Pebbles\",\n \"Color\",\n\t\"my_elastic_dataset\"\n ],\n\t\"type\": \"VIRTUAL_DATASET\",\n\t\"sql\": \"select * from review\",\n\t\"sqlContext\": [\"my_elasticsearch\", \"yelp\"]\n}"
headers = {
'Authorization': "_dremioo8opojj6vn4ughkvcpalpr46d6",
'Content-Type': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
Response
{
"entityType": "dataset",
"id": "e2676566-1946-4f11-8e21-9fbb366fa77a",
"type": "VIRTUAL_DATASET",
"path": [
"Pebbles",
"Color",
"my_elastic_dataset"
],
"createdAt": "2019-03-26T00:27:36.075Z",
"tag": "0",
"sql": "select * from review",
"sqlContext": [
"my_elasticsearch",
"yelp"
],
"accessControlList": {
"version": "0"
},
"fields": [
{
"name": "date",
"type": {
"name": "TIMESTAMP"
}
},
{
"name": "review_id",
"type": {
"name": "VARCHAR"
}
},
{
"name": "user_id",
"type": {
"name": "VARCHAR"
}
},
{
"name": "votes",
"type": {
"name": "STRUCT",
"subSchema": [
{
"name": "cool",
"type": {
"name": "BIGINT"
}
},
{
"name": "useful",
"type": {
"name": "BIGINT"
}
},
{
"name": "funny",
"type": {
"name": "BIGINT"
}
}
]
}
},
{
"name": "stars",
"type": {
"name": "DOUBLE"
}
},
{
"name": "text",
"type": {
"name": "VARCHAR"
}
},
{
"name": "type",
"type": {
"name": "VARCHAR"
}
},
{
"name": "business_id",
"type": {
"name": "VARCHAR"
}
},
{
"name": "_index",
"type": {
"name": "VARCHAR"
}
},
{
"name": "_type",
"type": {
"name": "VARCHAR"
}
},
{
"name": "_uid",
"type": {
"name": "VARCHAR"
}
}
]
}