Milvus-非结构化数据存储设计

此处使用官方SDK及Api

// 创建集合模式
CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder().enableDynamicField(false).build();

// 添加文档ID字段
schema.addField(AddFieldReq.builder()
        .fieldName("doc_id")
        .dataType(DataType.VarChar)
        .maxLength(256)
        .isPrimaryKey(true)
        .autoID(false)
        .build());

// 添加向量字段
schema.addField(AddFieldReq.builder()
        .fieldName("embedding")
        .dataType(DataType.FloatVector)
        .dimension(embeddingDimension)
        .build());

Map<String, Object> analyzerParams = new HashMap<>();
analyzerParams.put("type", "chinese");
schema.addField(AddFieldReq.builder()
        .fieldName("content")
        .dataType(DataType.VarChar)
        .maxLength(65535)
        .enableAnalyzer(true)
        .analyzerParams(analyzerParams)
        .enableMatch(true)
        .build());

schema.addField(AddFieldReq.builder()
        .fieldName("content_sparse")
        .dataType(DataType.SparseFloatVector)
        .build());

schema.addFunction(CreateCollectionReq.Function.builder()
        .functionType(FunctionType.BM25)
        .name("content_bm25_emb")
        .inputFieldNames(Collections.singletonList("content"))
        .outputFieldNames(Collections.singletonList("content_sparse"))
        .build());


schema.addField(AddFieldReq.builder()
        .fieldName("metadata")
        .dataType(DataType.JSON)
        .build());

// 创建索引
List<IndexParam> indexes = new ArrayList<>();



Map<String, Object> sparseParams = new HashMap<>();
sparseParams.put("inverted_index_algo", "DAAT_MAXSCORE");
indexes.add(IndexParam.builder()
        .fieldName("content_sparse")
        .indexName("content_sparse_index")
        .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
        .metricType(IndexParam.MetricType.BM25)
        .extraParams(sparseParams)
        .build());


HashMap<@Nullable String, @Nullable Object> extraParams = Maps.newHashMap();
extraParams.put("nlist", 1024);
indexes.add(IndexParam.builder()
        .fieldName("embedding")
        .indexName("embedding_index")
        .indexType(IndexParam.IndexType.IVF_FLAT)
        .metricType(IndexParam.MetricType.COSINE)
        .extraParams(extraParams)
        .build());

// 创建集合请求
CreateCollectionReq request = CreateCollectionReq.builder()
        .collectionName(collectionName)
        .collectionSchema(schema)
        .indexParams(indexes)
        .build();

// 创建集合
milvusClient.createCollection(request);

Schema如下