Dataset API

curl --location --request POST 'http://localhost:3000/api/core/dataset/create' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
  "parentId": null,
  "type": "dataset",
  "name":"Test",
  "intro":"Description",
  "avatar": "",
  "vectorModel": "text-embedding-ada-002",
  "agentModel": "gpt-3.5-turbo-16k",
  "vlmModel": "gpt-4.1"
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": "65abc9bd9d1448617cba5e6c"
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/list?parentId=' \
--header 'Authorization: Bearer xxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
    "parentId":""
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": [
    {
      "_id": "65abc9bd9d1448617cba5e6c",
      "parentId": null,
      "avatar": "",
      "name": "Test",
      "intro": "",
      "type": "dataset",
      "permission": "private",
      "canWrite": true,
      "isOwner": true,
      "vectorModel": {
        "model": "text-embedding-ada-002",
        "name": "Embedding-2",
        "charsPointsPrice": 0,
        "defaultToken": 512,
        "maxToken": 8000,
        "weight": 100
      }
    }
  ]
}

curl --location --request GET 'http://localhost:3000/api/core/dataset/detail?id=6593e137231a2be9c5603ba7' \
--header 'Authorization: Bearer {{authorization}}' \

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "_id": "6593e137231a2be9c5603ba7",
    "parentId": null,
    "teamId": "65422be6aa44b7da77729ec8",
    "tmbId": "65422be6aa44b7da77729ec9",
    "type": "dataset",
    "status": "active",
    "avatar": "/icon/logo.svg",
    "name": "FastGPT test",
    "vectorModel": {
      "model": "text-embedding-ada-002",
      "name": "Embedding-2",
      "charsPointsPrice": 0,
      "defaultToken": 512,
      "maxToken": 8000,
      "weight": 100
    },
    "agentModel": {
      "model": "gpt-3.5-turbo-16k",
      "name": "FastAI-16k",
      "maxContext": 16000,
      "maxResponse": 16000,
      "charsPointsPrice": 0
    },
    "intro": "",
    "permission": "private",
    "updateTime": "2024-01-02T10:11:03.084Z",
    "canWrite": true,
    "isOwner": true
  }
}

curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete?id=65abc8729d1448617cba5df6' \
--header 'Authorization: Bearer {{authorization}}' \

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": null
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "datasetId":"6593e137231a2be9c5603ba7",
    "parentId": null,
    "name":"Test",
    "type":"virtual",
    "metadata":{
      "test":111
    }
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": "65abcd009d1448617cba5ee1"
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/text' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "text":"xxxxxxxx",
    "datasetId":"6593e137231a2be9c5603ba7",
    "parentId": null,
    "name":"Test processing",

    "trainingType": "qa",
    "chunkSettingMode": "auto",
    "qaPrompt":"",

    "metadata":{}
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "collectionId": "65abcfab9d1448617cba5f0d",
    "results": {
      "insertLen": 5
    }
  }
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/link' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "link":"https://doc.fastgpt.io/guide/getting-started/quick-start",
    "datasetId":"6593e137231a2be9c5603ba7",
    "parentId": null,

    "trainingType": "chunk",
    "chunkSettingMode": "auto",
    "qaPrompt":"",

    "metadata":{
        "webPageSelector":".docs-content"
    }
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "collectionId": "65abd0ad9d1448617cba6031",
    "results": {
      "insertLen": 1
    }
  }
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/localFile' \
--header 'Authorization: Bearer {{authorization}}' \
--form 'file=@"C:\\Users\\user\\Desktop\\fastgpt测试文件\\index.html"' \
--form 'data="{\"datasetId\":\"6593e137231a2be9c5603ba7\",\"parentId\":null,\"trainingType\":\"chunk\",\"chunkSize\":512,\"chunkSplitter\":\"\",\"qaPrompt\":\"\",\"metadata\":{}}"'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "collectionId": "65abc044e4704bac793fbd81",
    "results": {
      "insertLen": 1
    }
  }
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/apiCollection' \
--header 'Authorization: Bearer fastgpt-xxx' \
--header 'Content-Type: application/json' \
--data-raw '{
  "name": "A Quick Guide to Building a Discord Bot.pdf",
  "apiFileId":"A Quick Guide to Building a Discord Bot.pdf",

  "datasetId": "674e9e479c3503c385495027",
  "parentId": null,

  "trainingType": "chunk",
  "chunkSize":512,
  "chunkSplitter":"",
  "qaPrompt":""
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "collectionId": "65abc044e4704bac793fbd81",
    "results": {
      "insertLen": 1
    }
  }
}

curl --location --request POST 'http://localhost:3000/api/proApi/core/dataset/collection/create/externalFileUrl' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'User-Agent: Apifox/1.0.0 (https://apifox.com)' \
--header 'Content-Type: application/json' \
--data-raw '{
    "externalFileUrl":"https://image.xxxxx.com/fastgpt-dev/%E6%91%82.pdf",
    "externalFileId":"1111",
    "createTime": "2024-05-01T00:00:00.000Z",
    "filename":"custom-file-name.pdf",
    "datasetId":"6642d105a5e9d2b00255b27b",
    "parentId": null,
    "tags": ["tag1","tag2"],

    "trainingType": "chunk",
    "chunkSize":512,
    "chunkSplitter":"",
    "qaPrompt":""
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "collectionId": "6646fcedfabd823cdc6de746",
    "results": {
      "insertLen": 1
    }
  }
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/listV2' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "offset":0,
    "pageSize": 10,
    "datasetId":"6593e137231a2be9c5603ba7",
    "parentId": null,
    "searchText":""
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "list": [
      {
        "_id": "6593e137231a2be9c5603ba9",
        "parentId": null,
        "tmbId": "65422be6aa44b7da77729ec9",
        "type": "virtual",
        "name": "Manual entry",
        "updateTime": "2099-01-01T00:00:00.000Z",
        "dataAmount": 3,
        "trainingAmount": 0,
        "externalFileId": "1111",
        "tags": ["11", "Test tag"],
        "forbid": false,
        "trainingType": "chunk",
        "permission": {
          "value": 4294967295,
          "isOwner": true,
          "hasManagePer": true,
          "hasWritePer": true,
          "hasReadPer": true
        }
      },
      {
        "_id": "65abd0ad9d1448617cba6031",
        "parentId": null,
        "tmbId": "65422be6aa44b7da77729ec9",
        "type": "link",
        "name": "Quick Start | FastGPT",
        "rawLink": "https://doc.fastgpt.io/guide/getting-started/quick-start",
        "updateTime": "2024-01-20T13:54:53.031Z",
        "dataAmount": 3,
        "trainingAmount": 0,
        "externalFileId": "222",
        "tags": ["Test tag"],
        "forbid": false,
        "trainingType": "chunk",
        "permission": {
          "value": 4294967295,
          "isOwner": true,
          "hasManagePer": true,
          "hasWritePer": true,
          "hasReadPer": true
        }
      }
    ],
    "total": 93
  }
}

curl --location --request GET 'http://localhost:3000/api/core/dataset/collection/detail?id=65abcfab9d1448617cba5f0d' \
--header 'Authorization: Bearer {{authorization}}' \

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "_id": "65abcfab9d1448617cba5f0d",
    "parentId": null,
    "teamId": "65422be6aa44b7da77729ec8",
    "tmbId": "65422be6aa44b7da77729ec9",
    "datasetId": {
      "_id": "6593e137231a2be9c5603ba7",
      "parentId": null,
      "teamId": "65422be6aa44b7da77729ec8",
      "tmbId": "65422be6aa44b7da77729ec9",
      "type": "dataset",
      "status": "active",
      "avatar": "/icon/logo.svg",
      "name": "FastGPT test",
      "vectorModel": "text-embedding-ada-002",
      "agentModel": "gpt-3.5-turbo-16k",
      "intro": "",
      "permission": "private",
      "updateTime": "2024-01-02T10:11:03.084Z"
    },
    "type": "virtual",
    "name": "Test processing",
    "trainingType": "qa",
    "chunkSize": 8000,
    "chunkSplitter": "",
    "qaPrompt": "11",
    "rawTextLength": 40466,
    "hashRawText": "47270840614c0cc122b29daaddc09c2a48f0ec6e77093611ab12b69cba7fee12",
    "createTime": "2024-01-20T13:50:35.838Z",
    "updateTime": "2024-01-20T13:50:35.838Z",
    "canWrite": true,
    "sourceName": "Test processing"
  }
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/update' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "id":"65abcfab9d1448617cba5f0d",
    "parentId": null,
    "name": "Test2222",
    "tags": ["tag1", "tag2"],
    "forbid": false,
    "createTime": "2024-01-01T00:00:00.000Z"
}'

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/update' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "datasetId":"6593e137231a2be9c5603ba7",
    "externalFileId":"1111",
    "parentId": null,
    "name": "Test2222",
    "tags": ["tag1", "tag2"],
    "forbid": false,
    "createTime": "2024-01-01T00:00:00.000Z"
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": null
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/delete' \
--header 'Authorization: Bearer fastgpt-' \
--header 'Content-Type: application/json' \
--data-raw '{
    "collectionIds": ["65a8cdcb0d70d3de0bf08d0a"]
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": null
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/data/pushData' \
--header 'Authorization: Bearer apikey' \
--header 'Content-Type: application/json' \
--data-raw '{
    "collectionId": "64663f451ba1676dbdef0499",
    "trainingType": "chunk",
    "prompt": "Optional. QA split guide prompt, ignored in chunk mode",
    "data": [
        {
            "q": "Who are you?",
            "a": "I am the FastGPT assistant"
        },
        {
            "q": "What can you do?",
            "a": "I can do anything",
            "indexes": [
                {
                    "text":"Custom index 1"
                },
                {
                    "text":"Custom index 2"
                }
            ]
        }
    ]
}'

{
  "code": 200,
  "statusText": "",
  "data": {
    "insertLen": 1
  }
}

Read the following text, [theme], and organize its information into questions and answers. Follow these requirements:
1. Create up to 25 questions.
2. Answer each question.
3. Answers should be detailed and complete, and can include plain text, links, code, tables, formulas, media links, and other markdown elements.
4. Return the questions and answers in this format:

Q1: Question.
A1: Answer.
Q2:
A2:
……

Text: """{{text}}"""

curl --location --request POST 'http://localhost:3000/api/core/dataset/data/v2/list' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "offset": 0,
    "pageSize": 10,
    "collectionId":"65abd4ac9d1448617cba6171",
    "searchText":""
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "list": [
      {
        "_id": "65abd4b29d1448617cba61db",
        "datasetId": "65abc9bd9d1448617cba5e6c",
        "collectionId": "65abd4ac9d1448617cba6171",
        "q": "No. 202212. China Academy of Information and Communications Technology; JD Explore Academy. September 2022. Artificial Intelligence Generated Content (AIGC) White Paper (2022). Copyright notice: Copyright in this white paper belongs to the China Academy of Information and Communications Technology and JD Explore Academy and is protected by law. Reprints, excerpts, and other uses of its text or views must credit the China Academy of Information and Communications Technology and JD Explore Academy as the source. The editors reserve the right to pursue legal responsibility for violations. Foreword: Xi Jinping has stated that digital technology, through new concepts, business forms, and models, is becoming fully integrated into every field and stage of economic, political, cultural, social, and ecological development. As the digital and physical worlds converge, artificial intelligence generated content (AIGC) is driving profound change in how digital content is produced and consumed. It will greatly enrich digital life and provide essential support for a new era of digital civilization.",
        "a": "",
        "chunkIndex": 0
      },
      {
        "_id": "65abd4b39d1448617cba624d",
        "datasetId": "65abc9bd9d1448617cba5e6c",
        "collectionId": "65abd4ac9d1448617cba6171",
        "q": "This white paper examines AIGC technology, applications, and governance. It outlines a technical framework covering both the digital representation and enhancement of real-world content and autonomous content creation using AI. It analyzes applications in media, e-commerce, film, and television, including emerging businesses and applications such as virtual digital humans and writing bots. From regulatory, technical, and enterprise perspectives, it examines challenges such as copyright disputes and the spread of misinformation. It concludes with recommendations for AIGC development and governance at the government, industry, enterprise, and societal levels. AI is evolving rapidly, and our understanding of AIGC requires further study. We welcome corrections and feedback on this white paper. Contents: I. The history and concept of AIGC ... 1; (1) History of AIGC ... 1; (2) The concept and scope of AIGC ... 4; II. The AIGC technology framework and its evolution ... 7; (1) AIGC technology advances ... 7; (2) The potential of large-model architectures for AIGC ... 10; (3) Three emerging capabilities in AIGC ... 18; III. AIGC applications ... 26; (1) AIGC + media: collaborative production by humans and machines,",
        "a": "",
        "chunkIndex": 1
      }
    ],
    "total": 63
  }
}

curl --location --request GET 'http://localhost:3000/api/core/dataset/data/detail?id=65abd4b29d1448617cba61db' \
--header 'Authorization: Bearer {{authorization}}' \

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": {
    "id": "65abd4b29d1448617cba61db",
    "q": "No. 202212. China Academy of Information and Communications Technology; JD Explore Academy. September 2022. Artificial Intelligence Generated Content (AIGC) White Paper (2022). Copyright notice: Copyright in this white paper belongs to the China Academy of Information and Communications Technology and JD Explore Academy and is protected by law. Reprints, excerpts, and other uses of its text or views must credit the China Academy of Information and Communications Technology and JD Explore Academy as the source. The editors reserve the right to pursue legal responsibility for violations. Foreword: Xi Jinping has stated that digital technology, through new concepts, business forms, and models, is becoming fully integrated into every field and stage of economic, political, cultural, social, and ecological development. As the digital and physical worlds converge, artificial intelligence generated content (AIGC) is driving profound change in how digital content is produced and consumed. It will greatly enrich digital life and provide essential support for a new era of digital civilization.",
    "a": "",
    "chunkIndex": 0,
    "indexes": [
      {
        "type": "default",
        "dataId": "3720083",
        "text": "No. 202212. China Academy of Information and Communications Technology; JD Explore Academy. September 2022. Artificial Intelligence Generated Content (AIGC) White Paper (2022). Copyright notice: Copyright in this white paper belongs to the China Academy of Information and Communications Technology and JD Explore Academy and is protected by law. Reprints, excerpts, and other uses of its text or views must credit the China Academy of Information and Communications Technology and JD Explore Academy as the source. The editors reserve the right to pursue legal responsibility for violations. Foreword: Xi Jinping has stated that digital technology, through new concepts, business forms, and models, is becoming fully integrated into every field and stage of economic, political, cultural, social, and ecological development. As the digital and physical worlds converge, artificial intelligence generated content (AIGC) is driving profound change in how digital content is produced and consumed. It will greatly enrich digital life and provide essential support for a new era of digital civilization.",
        "_id": "65abd4b29d1448617cba61dc"
      }
    ],
    "datasetId": "65abc9bd9d1448617cba5e6c",
    "collectionId": "65abd4ac9d1448617cba6171",
    "sourceName": "中文-AIGC白皮书2022.pdf",
    "sourceId": "65abd4ac9d1448617cba6166",
    "isOwner": true,
    "canWrite": true
  }
}

curl --location --request PUT 'http://localhost:3000/api/core/dataset/data/update' \
--header 'Authorization: Bearer {{authorization}}' \
--header 'Content-Type: application/json' \
--data-raw '{
    "dataId":"65abd4b29d1448617cba61db",
    "q":"Test 111",
    "a":"sss",
    "indexes":[
        {
            "dataId": "xxxx",
            "type": "default",
            "text": "Default index"
        },
        {
            "dataId": "xxx",
            "type": "custom",
            "text": "Existing custom index 1"
        },
        {
            "type":"custom",
            "text":"New custom index"
        }
    ]
}'

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": null
}

curl --location --request DELETE 'http://localhost:3000/api/core/dataset/data/delete?id=65abd4b39d1448617cba624d' \
--header 'Authorization: Bearer {{authorization}}' \

{
  "code": 200,
  "statusText": "",
  "message": "",
  "data": "success"
}

curl --location --request POST 'http://localhost:3000/api/core/dataset/searchTest' \
--header 'Authorization: Bearer fastgpt-xxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
    "datasetId": "Dataset ID",
    "text": "Who is the director",
    "limit": 5000,
    "similarity": 0,
    "searchMode": "embedding",
    "usingReRank": false,

    "datasetSearchUsingExtensionQuery": true,
    "datasetSearchExtensionModel": "gpt-5",
    "datasetSearchExtensionBg": ""
}'

{
  "code": 200,
  "statusText": "",
  "data": [
    {
      "id": "65599c54a5c814fb803363cb",
      "q": "Who are you?",
      "a": "I am the FastGPT assistant",
      "datasetId": "6554684f7f9ed18a39a4d15c",
      "collectionId": "6556cd795e4b663e770bb66d",
      "sourceName": "GBT 15104-2021 装饰单板贴面人造板.pdf",
      "sourceId": "6556cd775e4b663e770bb65c",
      "score": 0.8050316572189331
    }
  ]
}